From 822cd7d608ca70ead1fe17fc9e69ad0c82a447be Mon Sep 17 00:00:00 2001 From: Edoardo Comar Date: Mon, 5 Feb 2018 14:54:09 +0000 Subject: [PATCH 0001/1847] MINOR: Removed explicit class checks for PolicyViolationException (#4515) PolicyViolationException is a subclass of ApiException, and the handling was identical, hence explicit checking not required. --- core/src/main/scala/kafka/server/AdminManager.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala index 8264f7cdeda38..9014fab942df3 100644 --- a/core/src/main/scala/kafka/server/AdminManager.scala +++ b/core/src/main/scala/kafka/server/AdminManager.scala @@ -26,7 +26,7 @@ import kafka.utils._ import kafka.zk.{AdminZkClient, KafkaZkClient} import org.apache.kafka.clients.admin.NewPartitions import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, ConfigException, ConfigResource} -import org.apache.kafka.common.errors.{ApiException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, PolicyViolationException, ReassignmentInProgressException, UnknownTopicOrPartitionException} +import org.apache.kafka.common.errors.{ApiException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, UnknownTopicOrPartitionException} import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName @@ -128,7 +128,7 @@ class AdminManager(val config: KafkaConfig, CreatePartitionsMetadata(topic, assignments, ApiError.NONE) } catch { // Log client errors at a lower level than unexpected exceptions - case e@ (_: PolicyViolationException | _: ApiException) => + case e: ApiException => info(s"Error processing create topic request for topic $topic with arguments $arguments", e) CreatePartitionsMetadata(topic, Map(), ApiError.fromThrowable(e)) case e: Throwable => @@ -398,7 +398,7 @@ class AdminManager(val config: KafkaConfig, case e: Throwable => // Log client errors at a lower level than unexpected exceptions val message = s"Error processing alter configs request for resource $resource, config $config" - if (e.isInstanceOf[ApiException] || e.isInstanceOf[PolicyViolationException]) + if (e.isInstanceOf[ApiException]) info(message, e) else error(message, e) From eb3fef760e1c876b936f175e0eb9a1446cf5bdcf Mon Sep 17 00:00:00 2001 From: Jeff Klukas Date: Mon, 5 Feb 2018 09:46:07 -0800 Subject: [PATCH 0002/1847] KAFKA-6253: Improve sink connector topic regex validation KAFKA-3073 added topic regex support for sink connectors. The addition requires that you only specify one of topics or topics.regex settings. This is being validated in one place, but not during submission of connectors. This PR adds validation at `AbstractHerder.validateConnectorConfig` and `WorkerConnector.initialize`. This adds a test of the new behavior to `AbstractHerderTest`. Author: Jeff Klukas Reviewers: Randall Hauch , Ewen Cheslack-Postava Closes #4251 from jklukas/connect-topics-validation --- .../kafka/connect/runtime/AbstractHerder.java | 10 ++-- .../connect/runtime/SinkConnectorConfig.java | 33 ++++++++++++- .../connect/runtime/WorkerConnector.java | 3 ++ .../kafka/connect/runtime/WorkerSinkTask.java | 22 ++------- .../connect/runtime/AbstractHerderTest.java | 16 +++++++ .../distributed/DistributedHerderTest.java | 8 ++-- .../standalone/StandaloneHerderTest.java | 47 ++++++++++--------- 7 files changed, 92 insertions(+), 47 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java index 02465c922f106..b913f9ed65e25 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java @@ -256,9 +256,13 @@ public ConfigInfos validateConnectorConfig(Map connectorProps) { Connector connector = getConnector(connType); ClassLoader savedLoader = plugins().compareAndSwapLoaders(connector); try { - ConfigDef baseConfigDef = (connector instanceof SourceConnector) - ? SourceConnectorConfig.configDef() - : SinkConnectorConfig.configDef(); + ConfigDef baseConfigDef; + if (connector instanceof SourceConnector) { + baseConfigDef = SourceConnectorConfig.configDef(); + } else { + baseConfigDef = SinkConnectorConfig.configDef(); + SinkConnectorConfig.validate(connectorProps); + } ConfigDef enrichedConfigDef = ConnectorConfig.enrich(plugins(), baseConfigDef, connectorProps, false); Map validatedConnectorConfig = validateBasicConnectorConfig( connector, diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java index cf5564c25c245..887a4da2dea85 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java @@ -17,6 +17,7 @@ package org.apache.kafka.connect.runtime; import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.sink.SinkTask; import org.apache.kafka.connect.transforms.util.RegexValidator; @@ -34,7 +35,7 @@ public class SinkConnectorConfig extends ConnectorConfig { public static final String TOPICS_DEFAULT = ""; private static final String TOPICS_DISPLAY = "Topics"; - private static final String TOPICS_REGEX_CONFIG = SinkTask.TOPICS_REGEX_CONFIG; + public static final String TOPICS_REGEX_CONFIG = SinkTask.TOPICS_REGEX_CONFIG; private static final String TOPICS_REGEX_DOC = "Regular expression giving topics to consume. " + "Under the hood, the regex is compiled to a java.util.regex.Pattern. " + "Only one of " + TOPICS_CONFIG + " or " + TOPICS_REGEX_CONFIG + " should be specified."; @@ -52,4 +53,34 @@ public static ConfigDef configDef() { public SinkConnectorConfig(Plugins plugins, Map props) { super(plugins, config, props); } + + /** + * Throw an exception if the passed-in properties do not constitute a valid sink. + * @param props sink configuration properties + */ + public static void validate(Map props) { + final boolean hasTopicsConfig = hasTopicsConfig(props); + final boolean hasTopicsRegexConfig = hasTopicsRegexConfig(props); + + if (hasTopicsConfig && hasTopicsRegexConfig) { + throw new ConfigException(SinkTask.TOPICS_CONFIG + " and " + SinkTask.TOPICS_REGEX_CONFIG + + " are mutually exclusive options, but both are set."); + } + + if (!hasTopicsConfig && !hasTopicsRegexConfig) { + throw new ConfigException("Must configure one of " + + SinkTask.TOPICS_CONFIG + " or " + SinkTask.TOPICS_REGEX_CONFIG); + } + } + + public static boolean hasTopicsConfig(Map props) { + String topicsStr = props.get(TOPICS_CONFIG); + return topicsStr != null && !topicsStr.trim().isEmpty(); + } + + public static boolean hasTopicsRegexConfig(Map props) { + String topicsRegexStr = props.get(TOPICS_REGEX_CONFIG); + return topicsRegexStr != null && !topicsRegexStr.trim().isEmpty(); + } + } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java index 9b934f3428a74..611e196d9de9d 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java @@ -77,6 +77,9 @@ public void initialize(ConnectorConfig connectorConfig) { try { this.config = connectorConfig.originalsStrings(); log.debug("{} Initializing connector {} with config {}", this, connName, config); + if (isSinkConnector()) { + SinkConnectorConfig.validate(config); + } connector.initialize(new ConnectorContext() { @Override diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java index 85695bbe8baa7..5aeb851791ff2 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java @@ -25,7 +25,6 @@ import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -265,27 +264,14 @@ public int commitFailures() { * Initializes and starts the SinkTask. */ protected void initializeAndStart() { - String topicsStr = taskConfig.get(SinkTask.TOPICS_CONFIG); - boolean topicsStrPresent = topicsStr != null && !topicsStr.trim().isEmpty(); + SinkConnectorConfig.validate(taskConfig); - String topicsRegexStr = taskConfig.get(SinkTask.TOPICS_REGEX_CONFIG); - boolean topicsRegexStrPresent = topicsRegexStr != null && !topicsRegexStr.trim().isEmpty(); - - if (topicsStrPresent && topicsRegexStrPresent) { - throw new ConfigException(SinkTask.TOPICS_CONFIG + " and " + SinkTask.TOPICS_REGEX_CONFIG + - " are mutually exclusive options, but both are set."); - } - - if (!topicsStrPresent && !topicsRegexStrPresent) { - throw new ConfigException("Must configure one of " + - SinkTask.TOPICS_CONFIG + " or " + SinkTask.TOPICS_REGEX_CONFIG); - } - - if (topicsStrPresent) { - String[] topics = topicsStr.split(","); + if (SinkConnectorConfig.hasTopicsConfig(taskConfig)) { + String[] topics = taskConfig.get(SinkTask.TOPICS_CONFIG).split(","); consumer.subscribe(Arrays.asList(topics), new HandleRebalance()); log.debug("{} Initializing and starting task for topics {}", this, topics); } else { + String topicsRegexStr = taskConfig.get(SinkTask.TOPICS_REGEX_CONFIG); Pattern pattern = Pattern.compile(topicsRegexStr); consumer.subscribe(pattern, new HandleRebalance()); log.debug("{} Initializing and starting task for topics regex {}", this, topicsRegexStr); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java index dac1392ec7779..0718eb176532a 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.connect.runtime; import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.connect.connector.ConnectRecord; import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.runtime.isolation.PluginDesc; @@ -183,6 +184,21 @@ public void testConfigValidationMissingName() { verifyAll(); } + @Test(expected = ConfigException.class) + public void testConfigValidationInvalidTopics() { + AbstractHerder herder = createConfigValidationHerder(TestSinkConnector.class); + replayAll(); + + Map config = new HashMap(); + config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSinkConnector.class.getName()); + config.put(SinkConnectorConfig.TOPICS_CONFIG, "topic1,topic2"); + config.put(SinkConnectorConfig.TOPICS_REGEX_CONFIG, "topic.*"); + + herder.validateConnectorConfig(config); + + verifyAll(); + } + @Test() public void testConfigValidationTransformsExtendResults() { AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java index d7307cfdbc413..d7a7d87cb4fe4 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java @@ -355,7 +355,7 @@ public void testCreateConnector() throws Exception { PowerMock.expectLastCall(); // config validation - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); @@ -398,7 +398,7 @@ public void testCreateConnectorFailedBasicValidation() throws Exception { PowerMock.expectLastCall(); // config validation - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); @@ -443,7 +443,7 @@ public void testCreateConnectorFailedCustomValidation() throws Exception { PowerMock.expectLastCall(); // config validation - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); @@ -1338,7 +1338,7 @@ public void testPutConnectorConfig() throws Exception { PowerMock.expectLastCall(); // config validation - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes(); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java index 79be45b32901b..fd330f280098a 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java @@ -125,7 +125,8 @@ public void testCreateSourceConnector() throws Exception { expectAdd(SourceSink.SOURCE); Map config = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(config); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, config); PowerMock.replayAll(); @@ -142,7 +143,7 @@ public void testCreateConnectorFailedBasicValidation() throws Exception { Map config = connectorConfig(SourceSink.SOURCE); config.remove(ConnectorConfig.NAME_CONFIG); - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); @@ -167,7 +168,7 @@ public void testCreateConnectorFailedBasicValidation() throws Exception { public void testCreateConnectorFailedCustomValidation() throws Exception { connector = PowerMock.createMock(BogusSourceConnector.class); - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); @@ -199,7 +200,7 @@ public void testCreateConnectorAlreadyExists() throws Exception { expectAdd(SourceSink.SOURCE); Map config = connectorConfig(SourceSink.SOURCE); - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); expectConfigValidation(connectorMock, true, config, config); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(2); @@ -224,7 +225,8 @@ public void testCreateSinkConnector() throws Exception { expectAdd(SourceSink.SINK); Map config = connectorConfig(SourceSink.SINK); - expectConfigValidation(config); + Connector connectorMock = PowerMock.createMock(SinkConnector.class); + expectConfigValidation(connectorMock, true, config); PowerMock.replayAll(); herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback); @@ -238,7 +240,8 @@ public void testDestroyConnector() throws Exception { expectAdd(SourceSink.SOURCE); Map config = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(config); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, config); EasyMock.expect(statusBackingStore.getAll(CONNECTOR_NAME)).andReturn(Collections.emptyList()); statusBackingStore.put(new ConnectorStatus(CONNECTOR_NAME, AbstractStatus.State.DESTROYED, WORKER_ID, 0)); @@ -270,7 +273,8 @@ public void testRestartConnector() throws Exception { expectAdd(SourceSink.SOURCE); Map config = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(config); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, config); worker.stopConnector(CONNECTOR_NAME); EasyMock.expectLastCall().andReturn(true); @@ -295,7 +299,8 @@ public void testRestartConnectorFailureOnStart() throws Exception { expectAdd(SourceSink.SOURCE); Map config = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(config); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, config); worker.stopConnector(CONNECTOR_NAME); EasyMock.expectLastCall().andReturn(true); @@ -326,7 +331,8 @@ public void testRestartTask() throws Exception { expectAdd(SourceSink.SOURCE); Map connectorConfig = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(connectorConfig); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, connectorConfig); worker.stopAndAwaitTask(taskId); EasyMock.expectLastCall(); @@ -351,7 +357,8 @@ public void testRestartTaskFailureOnStart() throws Exception { expectAdd(SourceSink.SOURCE); Map connectorConfig = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(connectorConfig); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, connectorConfig); worker.stopAndAwaitTask(taskId); EasyMock.expectLastCall(); @@ -381,7 +388,8 @@ public void testCreateAndStop() throws Exception { expectAdd(SourceSink.SOURCE); Map connectorConfig = connectorConfig(SourceSink.SOURCE); - expectConfigValidation(connectorConfig); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connectorMock, true, connectorConfig); // herder.stop() should stop any running connectors and tasks even if destroyConnector was not invoked expectStop(); @@ -402,6 +410,7 @@ public void testCreateAndStop() throws Exception { @Test public void testAccessors() throws Exception { Map connConfig = connectorConfig(SourceSink.SOURCE); + System.out.println(connConfig); Callback> listConnectorsCb = PowerMock.createMock(Callback.class); Callback connectorInfoCb = PowerMock.createMock(Callback.class); @@ -421,7 +430,8 @@ public void testAccessors() throws Exception { // Create connector connector = PowerMock.createMock(BogusSourceConnector.class); expectAdd(SourceSink.SOURCE); - expectConfigValidation(connConfig); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); + expectConfigValidation(connector, true, connConfig); // Validate accessors with 1 connector listConnectorsCb.onCompletion(null, singleton(CONNECTOR_NAME)); @@ -467,7 +477,7 @@ public void testPutConnectorConfig() throws Exception { // Create connector = PowerMock.createMock(BogusSourceConnector.class); expectAdd(SourceSink.SOURCE); - Connector connectorMock = PowerMock.createMock(Connector.class); + Connector connectorMock = PowerMock.createMock(SourceConnector.class); expectConfigValidation(connectorMock, true, connConfig); // Should get first config @@ -526,7 +536,8 @@ public void testCorruptConfig() { Map config = new HashMap<>(); config.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME); config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, BogusSinkConnector.class.getName()); - Connector connectorMock = PowerMock.createMock(Connector.class); + config.put(SinkConnectorConfig.TOPICS_CONFIG, TOPICS_LIST_STR); + Connector connectorMock = PowerMock.createMock(SinkConnector.class); String error = "This is an error in your config!"; List errors = new ArrayList<>(singletonList(error)); String key = "foo.invalid.key"; @@ -592,7 +603,7 @@ private void expectAdd(SourceSink sourceSink) throws Exception { EasyMock.expect(herder.connectorTypeForClass(BogusSourceConnector.class.getName())) .andReturn(ConnectorType.SOURCE).anyTimes(); EasyMock.expect(herder.connectorTypeForClass(BogusSinkConnector.class.getName())) - .andReturn(ConnectorType.SINK).anyTimes(); + .andReturn(ConnectorType.SINK).anyTimes(); worker.isSinkConnector(CONNECTOR_NAME); PowerMock.expectLastCall().andReturn(sourceSink == SourceSink.SINK); } @@ -631,12 +642,6 @@ private static Map taskConfig(SourceSink sourceSink) { return generatedTaskProps; } - - private void expectConfigValidation(Map ... configs) { - Connector connectorMock = PowerMock.createMock(Connector.class); - expectConfigValidation(connectorMock, true, configs); - } - private void expectConfigValidation( Connector connectorMock, boolean shouldCreateConnector, From 7fe1c2b3d3a78ea3ffb9e269563653626861fbd2 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Mon, 5 Feb 2018 10:09:17 -0800 Subject: [PATCH 0003/1847] KAFKA-6254; Incremental fetch requests Author: Colin P. Mccabe Reviewers: Jason Gustafson , Ismael Juma , Jun Rao Closes #4418 from cmccabe/KAFKA-6254 --- .../kafka/clients/FetchSessionHandler.java | 443 +++++++++++ .../clients/consumer/internals/Fetcher.java | 88 ++- .../FetchSessionIdNotFoundException.java | 29 + .../InvalidFetchSessionEpochException.java | 29 + .../apache/kafka/common/protocol/Errors.java | 16 + .../kafka/common/protocol/types/Struct.java | 6 + .../kafka/common/requests/FetchMetadata.java | 154 ++++ .../kafka/common/requests/FetchRequest.java | 187 ++++- .../kafka/common/requests/FetchResponse.java | 79 +- .../common/utils/ImplicitLinkedHashSet.java | 354 +++++++++ .../clients/FetchSessionHandlerTest.java | 356 +++++++++ .../clients/consumer/KafkaConsumerTest.java | 3 +- .../consumer/internals/FetcherTest.java | 175 +++-- .../common/requests/RequestResponseTest.java | 70 +- .../utils/ImplicitLinkedHashSetTest.java | 239 ++++++ .../src/main/scala/kafka/api/ApiVersion.scala | 7 +- .../scala/kafka/server/FetchSession.scala | 720 ++++++++++++++++++ .../main/scala/kafka/server/KafkaApis.scala | 144 ++-- .../main/scala/kafka/server/KafkaConfig.scala | 15 + .../main/scala/kafka/server/KafkaServer.scala | 7 +- .../kafka/server/ReplicaFetcherThread.scala | 50 +- .../unit/kafka/server/FetchRequestTest.scala | 55 ++ .../unit/kafka/server/FetchSessionTest.scala | 312 ++++++++ .../unit/kafka/server/KafkaApisTest.scala | 2 + .../util/ReplicaFetcherMockBlockingSend.scala | 7 +- 25 files changed, 3329 insertions(+), 218 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/FetchSessionIdNotFoundException.java create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/InvalidFetchSessionEpochException.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/FetchMetadata.java create mode 100644 clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java create mode 100644 core/src/main/scala/kafka/server/FetchSession.scala create mode 100755 core/src/test/scala/unit/kafka/server/FetchSessionTest.scala diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java new file mode 100644 index 0000000000000..195324e833607 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.FetchMetadata; +import org.apache.kafka.common.requests.FetchRequest.PartitionData; +import org.apache.kafka.common.requests.FetchResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; + +/** + * FetchSessionHandler maintains the fetch session state for connecting to a broker. + * + * Using the protocol outlined by KIP-227, clients can create incremental fetch sessions. + * These sessions allow the client to fetch information about a set of partition over + * and over, without explicitly enumerating all the partitions in the request and the + * response. + * + * FetchSessionHandler tracks the partitions which are in the session. It also + * determines which partitions need to be included in each fetch request, and what + * the attached fetch session metadata should be for each request. The corresponding + * class on the receiving broker side is FetchManager. + */ +public class FetchSessionHandler { + private final Logger log; + + private final int node; + + /** + * The metadata for the next fetch request. + */ + private FetchMetadata nextMetadata = FetchMetadata.INITIAL; + + public FetchSessionHandler(LogContext logContext, int node) { + this.log = logContext.logger(FetchSessionHandler.class); + this.node = node; + } + + /** + * All of the partitions which exist in the fetch request session. + */ + private LinkedHashMap sessionPartitions = + new LinkedHashMap<>(0); + + public static class FetchRequestData { + /** + * The partitions to send in the fetch request. + */ + private final Map toSend; + + /** + * The partitions to send in the request's "forget" list. + */ + private final List toForget; + + /** + * All of the partitions which exist in the fetch request session. + */ + private final Map sessionPartitions; + + /** + * The metadata to use in this fetch request. + */ + private final FetchMetadata metadata; + + FetchRequestData(Map toSend, + List toForget, + Map sessionPartitions, + FetchMetadata metadata) { + this.toSend = toSend; + this.toForget = toForget; + this.sessionPartitions = sessionPartitions; + this.metadata = metadata; + } + + /** + * Get the set of partitions to send in this fetch request. + */ + public Map toSend() { + return toSend; + } + + /** + * Get a list of partitions to forget in this fetch request. + */ + public List toForget() { + return toForget; + } + + /** + * Get the full set of partitions involved in this fetch request. + */ + public Map sessionPartitions() { + return sessionPartitions; + } + + public FetchMetadata metadata() { + return metadata; + } + + @Override + public String toString() { + if (metadata.isFull()) { + StringBuilder bld = new StringBuilder("FullFetchRequest("); + String prefix = ""; + for (TopicPartition partition : toSend.keySet()) { + bld.append(prefix); + bld.append(partition); + prefix = ", "; + } + bld.append(")"); + return bld.toString(); + } else { + StringBuilder bld = new StringBuilder("IncrementalFetchRequest(toSend=("); + String prefix = ""; + for (TopicPartition partition : toSend.keySet()) { + bld.append(prefix); + bld.append(partition); + prefix = ", "; + } + bld.append("), toForget=("); + prefix = ""; + for (TopicPartition partition : toForget) { + bld.append(prefix); + bld.append(partition); + prefix = ", "; + } + bld.append("), implied=("); + prefix = ""; + for (TopicPartition partition : sessionPartitions.keySet()) { + if (!toSend.containsKey(partition)) { + bld.append(prefix); + bld.append(partition); + prefix = ", "; + } + } + bld.append("))"); + return bld.toString(); + } + } + } + + public class Builder { + /** + * The next partitions which we want to fetch. + * + * It is important to maintain the insertion order of this list by using a LinkedHashMap rather + * than a regular Map. + * + * One reason is that when dealing with FULL fetch requests, if there is not enough response + * space to return data from all partitions, the server will only return data from partitions + * early in this list. + * + * Another reason is because we make use of the list ordering to optimize the preparation of + * incremental fetch requests (see below). + */ + private LinkedHashMap next = new LinkedHashMap<>(); + + /** + * Mark that we want data from this partition in the upcoming fetch. + */ + public void add(TopicPartition topicPartition, PartitionData data) { + next.put(topicPartition, data); + } + + public FetchRequestData build() { + if (nextMetadata.isFull()) { + log.debug("Built full fetch {} for node {} with {}.", + nextMetadata, node, partitionsToLogString(next.keySet())); + sessionPartitions = next; + next = null; + Map toSend = + Collections.unmodifiableMap(new LinkedHashMap<>(sessionPartitions)); + return new FetchRequestData(toSend, Collections.emptyList(), toSend, nextMetadata); + } + + List added = new ArrayList<>(); + List removed = new ArrayList<>(); + List altered = new ArrayList<>(); + for (Iterator> iter = + sessionPartitions.entrySet().iterator(); iter.hasNext(); ) { + Entry entry = iter.next(); + TopicPartition topicPartition = entry.getKey(); + PartitionData prevData = entry.getValue(); + PartitionData nextData = next.get(topicPartition); + if (nextData != null) { + if (prevData.equals(nextData)) { + // Omit this partition from the FetchRequest, because it hasn't changed + // since the previous request. + next.remove(topicPartition); + } else { + // Move the altered partition to the end of 'next' + next.remove(topicPartition); + next.put(topicPartition, nextData); + entry.setValue(nextData); + altered.add(topicPartition); + } + } else { + // Remove this partition from the session. + iter.remove(); + // Indicate that we no longer want to listen to this partition. + removed.add(topicPartition); + } + } + // Add any new partitions to the session. + for (Iterator> iter = + next.entrySet().iterator(); iter.hasNext(); ) { + Entry entry = iter.next(); + TopicPartition topicPartition = entry.getKey(); + PartitionData nextData = entry.getValue(); + if (sessionPartitions.containsKey(topicPartition)) { + // In the previous loop, all the partitions which existed in both sessionPartitions + // and next were moved to the end of next, or removed from next. Therefore, + // once we hit one of them, we know there are no more unseen entries to look + // at in next. + break; + } + sessionPartitions.put(topicPartition, nextData); + added.add(topicPartition); + } + log.debug("Built incremental fetch {} for node {}. Added {}, altered {}, removed {} " + + "out of {}", nextMetadata, node, partitionsToLogString(added), + partitionsToLogString(altered), partitionsToLogString(removed), + partitionsToLogString(sessionPartitions.keySet())); + Map toSend = + Collections.unmodifiableMap(new LinkedHashMap<>(next)); + Map curSessionPartitions = + Collections.unmodifiableMap(new LinkedHashMap<>(sessionPartitions)); + next = null; + return new FetchRequestData(toSend, Collections.unmodifiableList(removed), + curSessionPartitions, nextMetadata); + } + } + + public Builder newBuilder() { + return new Builder(); + } + + private String partitionsToLogString(Collection partitions) { + if (!log.isTraceEnabled()) { + return String.format("%d partition(s)", partitions.size()); + } + return "(" + Utils.join(partitions, ", ") + ")"; + } + + /** + * Return some partitions which are expected to be in a particular set, but which are not. + * + * @param toFind The partitions to look for. + * @param toSearch The set of partitions to search. + * @return null if all partitions were found; some of the missing ones + * in string form, if not. + */ + static Set findMissing(Set toFind, Set toSearch) { + Set ret = new LinkedHashSet<>(); + for (TopicPartition partition : toFind) { + if (!toSearch.contains(partition)) { + ret.add(partition); + } + } + return ret; + } + + /** + * Verify that a full fetch response contains all the partitions in the fetch session. + * + * @param response The response. + * @return True if the full fetch response partitions are valid. + */ + private String verifyFullFetchResponsePartitions(FetchResponse response) { + StringBuilder bld = new StringBuilder(); + Set omitted = + findMissing(response.responseData().keySet(), sessionPartitions.keySet()); + Set extra = + findMissing(sessionPartitions.keySet(), response.responseData().keySet()); + if (!omitted.isEmpty()) { + bld.append("omitted=(").append(Utils.join(omitted, ", ")).append(", "); + } + if (!extra.isEmpty()) { + bld.append("extra=(").append(Utils.join(extra, ", ")).append(", "); + } + if ((!omitted.isEmpty()) || (!extra.isEmpty())) { + bld.append("response=(").append(Utils.join(response.responseData().keySet(), ", ")); + return bld.toString(); + } + return null; + } + + /** + * Verify that the partitions in an incremental fetch response are contained in the session. + * + * @param response The response. + * @return True if the incremental fetch response partitions are valid. + */ + private String verifyIncrementalFetchResponsePartitions(FetchResponse response) { + Set extra = + findMissing(response.responseData().keySet(), sessionPartitions.keySet()); + if (!extra.isEmpty()) { + StringBuilder bld = new StringBuilder(); + bld.append("extra=(").append(Utils.join(extra, ", ")).append("), "); + bld.append("response=(").append( + Utils.join(response.responseData().keySet(), ", ")).append("), "); + return bld.toString(); + } + return null; + } + + /** + * Create a string describing the partitions in a FetchResponse. + * + * @param response The FetchResponse. + * @return The string to log. + */ + private String responseDataToLogString(FetchResponse response) { + if (!log.isTraceEnabled()) { + int implied = sessionPartitions.size() - response.responseData().size(); + if (implied > 0) { + return String.format(" with %d response partition(s), %d implied partition(s)", + response.responseData().size(), implied); + } else { + return String.format(" with %d response partition(s)", + response.responseData().size()); + } + } + StringBuilder bld = new StringBuilder(); + bld.append(" with response=("). + append(Utils.join(response.responseData().keySet(), ", ")). + append(")"); + String prefix = ", implied=("; + String suffix = ""; + for (TopicPartition partition : sessionPartitions.keySet()) { + if (!response.responseData().containsKey(partition)) { + bld.append(prefix); + bld.append(partition); + prefix = ", "; + suffix = ")"; + } + } + bld.append(suffix); + return bld.toString(); + } + + /** + * Handle the fetch response. + * + * @param response The response. + * @return True if the response is well-formed; false if it can't be processed + * because of missing or unexpected partitions. + */ + public boolean handleResponse(FetchResponse response) { + if (response.error() != Errors.NONE) { + log.info("Node {} was unable to process the fetch request with {}: {}.", + node, nextMetadata, response.error()); + if (response.error() == Errors.FETCH_SESSION_ID_NOT_FOUND) { + nextMetadata = FetchMetadata.INITIAL; + } else { + nextMetadata = nextMetadata.nextCloseExisting(); + } + return false; + } else if (nextMetadata.isFull()) { + String problem = verifyFullFetchResponsePartitions(response); + if (problem != null) { + log.info("Node {} sent an invalid full fetch response with {}", node, problem); + nextMetadata = FetchMetadata.INITIAL; + return false; + } else if (response.sessionId() == INVALID_SESSION_ID) { + log.debug("Node {} sent a full fetch response{}", + node, responseDataToLogString(response)); + nextMetadata = FetchMetadata.INITIAL; + return true; + } else { + // The server created a new incremental fetch session. + log.debug("Node {} sent a full fetch response that created a new incremental " + + "fetch session {}{}", node, response.sessionId(), responseDataToLogString(response)); + nextMetadata = FetchMetadata.newIncremental(response.sessionId()); + return true; + } + } else { + String problem = verifyIncrementalFetchResponsePartitions(response); + if (problem != null) { + log.info("Node {} sent an invalid incremental fetch response with {}", node, problem); + nextMetadata = nextMetadata.nextCloseExisting(); + return false; + } else if (response.sessionId() == INVALID_SESSION_ID) { + // The incremental fetch session was closed by the server. + log.debug("Node {} sent an incremental fetch response closing session {}{}", + node, nextMetadata.sessionId(), responseDataToLogString(response)); + nextMetadata = FetchMetadata.INITIAL; + return true; + } else { + // The incremental fetch session was continued by the server. + log.debug("Node {} sent an incremental fetch response for session {}{}", + node, response.sessionId(), responseDataToLogString(response)); + nextMetadata = nextMetadata.nextIncremental(); + return true; + } + } + } + + /** + * Handle an error sending the prepared request. + * + * When a network error occurs, we close any existing fetch session on our next request, + * and try to create a new session. + * + * @param t The exception. + */ + public void handleError(Throwable t) { + log.info("Error sending fetch request {} to node {}: {}.", nextMetadata, node, t.toString()); + nextMetadata = nextMetadata.nextCloseExisting(); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 6d56139118a28..32782ee53db70 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.FetchSessionHandler; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -92,6 +93,7 @@ */ public class Fetcher implements SubscriptionState.Listener, Closeable { private final Logger log; + private final LogContext logContext; private final ConsumerNetworkClient client; private final Time time; private final int minBytes; @@ -110,6 +112,7 @@ public class Fetcher implements SubscriptionState.Listener, Closeable { private final ExtendedDeserializer keyDeserializer; private final ExtendedDeserializer valueDeserializer; private final IsolationLevel isolationLevel; + private final Map sessionHandlers; private PartitionRecords nextInLineRecords = null; @@ -131,6 +134,7 @@ public Fetcher(LogContext logContext, long retryBackoffMs, IsolationLevel isolationLevel) { this.log = logContext.logger(Fetcher.class); + this.logContext = logContext; this.time = time; this.client = client; this.metadata = metadata; @@ -147,6 +151,7 @@ public Fetcher(LogContext logContext, this.sensors = new FetchManagerMetrics(metrics, metricsRegistry); this.retryBackoffMs = retryBackoffMs; this.isolationLevel = isolationLevel; + this.sessionHandlers = new HashMap<>(); subscriptions.addListener(this); } @@ -181,36 +186,37 @@ public boolean hasCompletedFetches() { return !completedFetches.isEmpty(); } - private boolean matchesRequestedPartitions(FetchRequest.Builder request, FetchResponse response) { - Set requestedPartitions = request.fetchData().keySet(); - Set fetchedPartitions = response.responseData().keySet(); - return fetchedPartitions.equals(requestedPartitions); - } - /** * Set-up a fetch request for any node that we have assigned partitions for which doesn't already have * an in-flight fetch or pending fetch data. * @return number of fetches sent */ public int sendFetches() { - Map fetchRequestMap = createFetchRequests(); - for (Map.Entry fetchEntry : fetchRequestMap.entrySet()) { - final FetchRequest.Builder request = fetchEntry.getValue(); - final Node fetchTarget = fetchEntry.getKey(); - - log.debug("Sending {} fetch for partitions {} to broker {}", isolationLevel, request.fetchData().keySet(), - fetchTarget); + Map fetchRequestMap = prepareFetchRequests(); + for (Map.Entry entry : fetchRequestMap.entrySet()) { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = FetchRequest.Builder. + forConsumer(this.maxWaitMs, this.minBytes, data.toSend()) + .isolationLevel(isolationLevel) + .setMaxBytes(this.maxBytes) + .metadata(data.metadata()) + .toForget(data.toForget()); + if (log.isDebugEnabled()) { + log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget); + } client.send(fetchTarget, request) .addListener(new RequestFutureListener() { @Override public void onSuccess(ClientResponse resp) { FetchResponse response = (FetchResponse) resp.responseBody(); - if (!matchesRequestedPartitions(request, response)) { - // obviously we expect the broker to always send us valid responses, so this check - // is mainly for test cases where mock fetch responses must be manually crafted. - log.warn("Ignoring fetch response containing partitions {} since it does not match " + - "the requested partitions {}", response.responseData().keySet(), - request.fetchData().keySet()); + FetchSessionHandler handler = sessionHandlers.get(fetchTarget.id()); + if (handler == null) { + log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.", + fetchTarget.id()); + return; + } + if (!handler.handleResponse(response)) { return; } @@ -219,7 +225,7 @@ public void onSuccess(ClientResponse resp) { for (Map.Entry entry : response.responseData().entrySet()) { TopicPartition partition = entry.getKey(); - long fetchOffset = request.fetchData().get(partition).fetchOffset; + long fetchOffset = data.sessionPartitions().get(partition).fetchOffset; FetchResponse.PartitionData fetchData = entry.getValue(); log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", @@ -233,7 +239,10 @@ public void onSuccess(ClientResponse resp) { @Override public void onFailure(RuntimeException e) { - log.debug("Fetch request {} to {} failed", request.fetchData(), fetchTarget, e); + FetchSessionHandler handler = sessionHandlers.get(fetchTarget.id()); + if (handler != null) { + handler.handleError(e); + } } }); } @@ -772,42 +781,41 @@ private List fetchablePartitions() { * Create fetch requests for all nodes for which we have assigned partitions * that have no existing requests in flight. */ - private Map createFetchRequests() { + private Map prepareFetchRequests() { // create the fetch info Cluster cluster = metadata.fetch(); - Map> fetchable = new LinkedHashMap<>(); + Map fetchable = new LinkedHashMap<>(); for (TopicPartition partition : fetchablePartitions()) { Node node = cluster.leaderFor(partition); if (node == null) { metadata.requestUpdate(); } else if (!this.client.hasPendingRequests(node)) { // if there is a leader and no in-flight requests, issue a new fetch - LinkedHashMap fetch = fetchable.get(node); - if (fetch == null) { - fetch = new LinkedHashMap<>(); - fetchable.put(node, fetch); + FetchSessionHandler.Builder builder = fetchable.get(node); + if (builder == null) { + FetchSessionHandler handler = sessionHandlers.get(node.id()); + if (handler == null) { + handler = new FetchSessionHandler(logContext, node.id()); + sessionHandlers.put(node.id(), handler); + } + builder = handler.newBuilder(); + fetchable.put(node, builder); } long position = this.subscriptions.position(partition); - fetch.put(partition, new FetchRequest.PartitionData(position, FetchRequest.INVALID_LOG_START_OFFSET, - this.fetchSize)); + builder.add(partition, new FetchRequest.PartitionData(position, FetchRequest.INVALID_LOG_START_OFFSET, + this.fetchSize)); log.debug("Added {} fetch request for partition {} at offset {} to node {}", isolationLevel, - partition, position, node); + partition, position, node); } else { log.trace("Skipping fetch for partition {} because there is an in-flight request to {}", partition, node); } } - - // create the fetches - Map requests = new HashMap<>(); - for (Map.Entry> entry : fetchable.entrySet()) { - Node node = entry.getKey(); - FetchRequest.Builder fetch = FetchRequest.Builder.forConsumer(this.maxWaitMs, this.minBytes, - entry.getValue(), isolationLevel) - .setMaxBytes(this.maxBytes); - requests.put(node, fetch); + Map reqs = new LinkedHashMap<>(); + for (Map.Entry entry : fetchable.entrySet()) { + reqs.put(entry.getKey(), entry.getValue().build()); } - return requests; + return reqs; } /** diff --git a/clients/src/main/java/org/apache/kafka/common/errors/FetchSessionIdNotFoundException.java b/clients/src/main/java/org/apache/kafka/common/errors/FetchSessionIdNotFoundException.java new file mode 100644 index 0000000000000..2ce5f740d6719 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/FetchSessionIdNotFoundException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.errors; + +public class FetchSessionIdNotFoundException extends RetriableException { + private static final long serialVersionUID = 1L; + + public FetchSessionIdNotFoundException() { + } + + public FetchSessionIdNotFoundException(String message) { + super(message); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/errors/InvalidFetchSessionEpochException.java b/clients/src/main/java/org/apache/kafka/common/errors/InvalidFetchSessionEpochException.java new file mode 100644 index 0000000000000..3b135c0147d06 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/InvalidFetchSessionEpochException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.errors; + +public class InvalidFetchSessionEpochException extends RetriableException { + private static final long serialVersionUID = 1L; + + public InvalidFetchSessionEpochException() { + } + + public InvalidFetchSessionEpochException(String message) { + super(message); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index e2b8aeaf709fe..4b44c18430fc9 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -30,6 +30,7 @@ import org.apache.kafka.common.errors.DelegationTokenExpiredException; import org.apache.kafka.common.errors.DelegationTokenNotFoundException; import org.apache.kafka.common.errors.DelegationTokenOwnerMismatchException; +import org.apache.kafka.common.errors.FetchSessionIdNotFoundException; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.GroupIdNotFoundException; import org.apache.kafka.common.errors.GroupNotEmptyException; @@ -38,6 +39,7 @@ import org.apache.kafka.common.errors.InconsistentGroupProtocolException; import org.apache.kafka.common.errors.InvalidCommitOffsetSizeException; import org.apache.kafka.common.errors.InvalidConfigurationException; +import org.apache.kafka.common.errors.InvalidFetchSessionEpochException; import org.apache.kafka.common.errors.InvalidFetchSizeException; import org.apache.kafka.common.errors.InvalidGroupIdException; import org.apache.kafka.common.errors.InvalidPartitionsException; @@ -608,6 +610,20 @@ public ApiException build(String message) { public ApiException build(String message) { return new GroupIdNotFoundException(message); } + }), + FETCH_SESSION_ID_NOT_FOUND(70, "The fetch session ID was not found", + new ApiExceptionBuilder() { + @Override + public ApiException build(String message) { + return new FetchSessionIdNotFoundException(message); + } + }), + INVALID_FETCH_SESSION_EPOCH(71, "The fetch session epoch is invalid", + new ApiExceptionBuilder() { + @Override + public ApiException build(String message) { + return new InvalidFetchSessionEpochException(message); + } }); private interface ApiExceptionBuilder { diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java index 6fb6b20ca311d..ac24a1b69b2b3 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java @@ -105,6 +105,12 @@ public Long getOrElse(Field.Int64 field, long alternative) { return alternative; } + public Short getOrElse(Field.Int16 field, short alternative) { + if (hasField(field.name)) + return getShort(field.name); + return alternative; + } + public Integer getOrElse(Field.Int32 field, int alternative) { if (hasField(field.name)) return getInt(field.name); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchMetadata.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchMetadata.java new file mode 100644 index 0000000000000..feb6953f9dafd --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchMetadata.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.requests; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +public class FetchMetadata { + public static final Logger log = LoggerFactory.getLogger(FetchMetadata.class); + + /** + * The session ID used by clients with no session. + */ + public static final int INVALID_SESSION_ID = 0; + + /** + * The first epoch. When used in a fetch request, indicates that the client + * wants to create or recreate a session. + */ + public static final int INITIAL_EPOCH = 0; + + /** + * An invalid epoch. When used in a fetch request, indicates that the client + * wants to close any existing session, and not create a new one. + */ + public static final int FINAL_EPOCH = -1; + + /** + * The FetchMetadata that is used when initializing a new FetchSessionHandler. + */ + public static final FetchMetadata INITIAL = new FetchMetadata(INVALID_SESSION_ID, INITIAL_EPOCH); + + /** + * The FetchMetadata that is implicitly used for handling older FetchRequests that + * don't include fetch metadata. + */ + public static final FetchMetadata LEGACY = new FetchMetadata(INVALID_SESSION_ID, FINAL_EPOCH); + + /** + * Returns the next epoch. + * + * @param prevEpoch The previous epoch. + * @return The next epoch. + */ + public static int nextEpoch(int prevEpoch) { + if (prevEpoch < 0) { + // The next epoch after FINAL_EPOCH is always FINAL_EPOCH itself. + return FINAL_EPOCH; + } else if (prevEpoch == Integer.MAX_VALUE) { + return 1; + } else { + return prevEpoch + 1; + } + } + + /** + * The fetch session ID. + */ + private final int sessionId; + + /** + * The fetch session epoch. + */ + private final int epoch; + + public FetchMetadata(int sessionId, int epoch) { + this.sessionId = sessionId; + this.epoch = epoch; + } + + /** + * Returns true if this is a full fetch request. + */ + public boolean isFull() { + return (this.epoch == INITIAL_EPOCH) || (this.epoch == FINAL_EPOCH); + } + + public int sessionId() { + return sessionId; + } + + public int epoch() { + return epoch; + } + + @Override + public int hashCode() { + return Objects.hash(sessionId, epoch); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FetchMetadata that = (FetchMetadata) o; + return sessionId == that.sessionId && epoch == that.epoch; + } + + /** + * Return the metadata for the next error response. + */ + public FetchMetadata nextCloseExisting() { + return new FetchMetadata(sessionId, INITIAL_EPOCH); + } + + /** + * Return the metadata for the next full fetch request. + */ + public static FetchMetadata newIncremental(int sessionId) { + return new FetchMetadata(sessionId, nextEpoch(INITIAL_EPOCH)); + } + + /** + * Return the metadata for the next incremental response. + */ + public FetchMetadata nextIncremental() { + return new FetchMetadata(sessionId, nextEpoch(epoch)); + } + + @Override + public String toString() { + StringBuilder bld = new StringBuilder(); + if (sessionId == INVALID_SESSION_ID) { + bld.append("(sessionId=INVALID, "); + } else { + bld.append("(sessionId=").append(sessionId).append(", "); + } + if (epoch == INITIAL_EPOCH) { + bld.append("epoch=INITIAL)"); + } else if (epoch == FINAL_EPOCH) { + bld.append("epoch=FINAL)"); + } else { + bld.append("epoch=").append(epoch).append(")"); + } + return bld.toString(); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 18425d0360ba8..65cf7fea6b1d7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -23,19 +23,27 @@ import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.types.Type; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID; import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME; import static org.apache.kafka.common.protocol.types.Type.INT32; import static org.apache.kafka.common.protocol.types.Type.INT64; import static org.apache.kafka.common.protocol.types.Type.INT8; +import static org.apache.kafka.common.requests.FetchMetadata.FINAL_EPOCH; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; public class FetchRequest extends AbstractRequest { public static final int CONSUMER_REPLICA_ID = -1; @@ -44,6 +52,7 @@ public class FetchRequest extends AbstractRequest { private static final String MIN_BYTES_KEY_NAME = "min_bytes"; private static final String ISOLATION_LEVEL_KEY_NAME = "isolation_level"; private static final String TOPICS_KEY_NAME = "topics"; + private static final String FORGOTTEN_TOPICS_DATA = "forgetten_topics_data"; // request and partition level name private static final String MAX_BYTES_KEY_NAME = "max_bytes"; @@ -139,9 +148,36 @@ public class FetchRequest extends AbstractRequest { */ private static final Schema FETCH_REQUEST_V6 = FETCH_REQUEST_V5; + // FETCH_REQUEST_V7 added incremental fetch requests. + public static final Field.Int32 SESSION_ID = new Field.Int32("session_id", "The fetch session ID"); + public static final Field.Int32 EPOCH = new Field.Int32("epoch", "The fetch epoch"); + + private static final Schema FORGOTTEN_TOPIC_DATA = new Schema( + TOPIC_NAME, + new Field(PARTITIONS_KEY_NAME, new ArrayOf(Type.INT32), + "Partitions to remove from the fetch session.")); + + private static final Schema FETCH_REQUEST_V7 = new Schema( + new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."), + new Field(MAX_WAIT_KEY_NAME, INT32, "Maximum time in ms to wait for the response."), + new Field(MIN_BYTES_KEY_NAME, INT32, "Minimum bytes to accumulate in the response."), + new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, " + + "if the first message in the first non-empty partition of the fetch is larger than this " + + "value, the message will still be returned to ensure that progress can be made."), + new Field(ISOLATION_LEVEL_KEY_NAME, INT8, "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED " + + "(isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), " + + "non-transactional and COMMITTED transactional records are visible. To be more concrete, " + + "READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), " + + "and enables the inclusion of the list of aborted transactions in the result, which allows " + + "consumers to discard ABORTED transactional records"), + SESSION_ID, + EPOCH, + new Field(TOPICS_KEY_NAME, new ArrayOf(FETCH_REQUEST_TOPIC_V5), "Topics to fetch in the order provided."), + new Field(FORGOTTEN_TOPICS_DATA, new ArrayOf(FORGOTTEN_TOPIC_DATA), "Topics to remove from the fetch session.")); + public static Schema[] schemaVersions() { return new Schema[]{FETCH_REQUEST_V0, FETCH_REQUEST_V1, FETCH_REQUEST_V2, FETCH_REQUEST_V3, FETCH_REQUEST_V4, - FETCH_REQUEST_V5, FETCH_REQUEST_V6}; + FETCH_REQUEST_V5, FETCH_REQUEST_V6, FETCH_REQUEST_V7}; }; // default values for older versions where a request level limit did not exist @@ -153,7 +189,14 @@ public static Schema[] schemaVersions() { private final int minBytes; private final int maxBytes; private final IsolationLevel isolationLevel; - private final LinkedHashMap fetchData; + + // Note: the iteration order of this map is significant, since it determines the order + // in which partitions appear in the message. For this reason, this map should have a + // deterministic iteration order, like LinkedHashMap or TreeMap (but unlike HashMap). + private final Map fetchData; + + private final List toForget; + private final FetchMetadata metadata; public static final class PartitionData { public final long fetchOffset; @@ -170,6 +213,21 @@ public PartitionData(long fetchOffset, long logStartOffset, int maxBytes) { public String toString() { return "(offset=" + fetchOffset + ", logStartOffset=" + logStartOffset + ", maxBytes=" + maxBytes + ")"; } + + @Override + public int hashCode() { + return Objects.hash(fetchOffset, logStartOffset, maxBytes); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PartitionData that = (PartitionData) o; + return Objects.equals(fetchOffset, that.fetchOffset) && + Objects.equals(logStartOffset, that.logStartOffset) && + Objects.equals(maxBytes, that.maxBytes); + } } static final class TopicAndPartitionData { @@ -181,9 +239,10 @@ public TopicAndPartitionData(String topic) { this.partitions = new LinkedHashMap<>(); } - public static List> batchByTopic(LinkedHashMap data) { + public static List> batchByTopic(Iterator> iter) { List> topics = new ArrayList<>(); - for (Map.Entry topicEntry : data.entrySet()) { + while (iter.hasNext()) { + Map.Entry topicEntry = iter.next(); String topic = topicEntry.getKey().topic(); int partition = topicEntry.getKey().partition(); T partitionData = topicEntry.getValue(); @@ -199,37 +258,42 @@ public static class Builder extends AbstractRequest.Builder { private final int maxWait; private final int minBytes; private final int replicaId; - private final LinkedHashMap fetchData; - private final IsolationLevel isolationLevel; + private final Map fetchData; + private IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED; private int maxBytes = DEFAULT_RESPONSE_MAX_BYTES; + private FetchMetadata metadata = FetchMetadata.LEGACY; + private List toForget = Collections.emptyList(); - public static Builder forConsumer(int maxWait, int minBytes, LinkedHashMap fetchData) { - return forConsumer(maxWait, minBytes, fetchData, IsolationLevel.READ_UNCOMMITTED); - } - - public static Builder forConsumer(int maxWait, int minBytes, LinkedHashMap fetchData, - IsolationLevel isolationLevel) { - return new Builder(ApiKeys.FETCH.oldestVersion(), ApiKeys.FETCH.latestVersion(), CONSUMER_REPLICA_ID, - maxWait, minBytes, fetchData, isolationLevel); + public static Builder forConsumer(int maxWait, int minBytes, Map fetchData) { + return new Builder(ApiKeys.FETCH.oldestVersion(), ApiKeys.FETCH.latestVersion(), + CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData); } public static Builder forReplica(short allowedVersion, int replicaId, int maxWait, int minBytes, - LinkedHashMap fetchData) { - return new Builder(allowedVersion, allowedVersion, replicaId, maxWait, minBytes, fetchData, - IsolationLevel.READ_UNCOMMITTED); + Map fetchData) { + return new Builder(allowedVersion, allowedVersion, replicaId, maxWait, minBytes, fetchData); } - private Builder(short minVersion, short maxVersion, int replicaId, int maxWait, int minBytes, - LinkedHashMap fetchData, IsolationLevel isolationLevel) { + public Builder(short minVersion, short maxVersion, int replicaId, int maxWait, int minBytes, + Map fetchData) { super(ApiKeys.FETCH, minVersion, maxVersion); this.replicaId = replicaId; this.maxWait = maxWait; this.minBytes = minBytes; this.fetchData = fetchData; + } + + public Builder isolationLevel(IsolationLevel isolationLevel) { this.isolationLevel = isolationLevel; + return this; + } + + public Builder metadata(FetchMetadata metadata) { + this.metadata = metadata; + return this; } - public LinkedHashMap fetchData() { + public Map fetchData() { return this.fetchData; } @@ -238,13 +302,23 @@ public Builder setMaxBytes(int maxBytes) { return this; } + public List toForget() { + return toForget; + } + + public Builder toForget(List toForget) { + this.toForget = toForget; + return this; + } + @Override public FetchRequest build(short version) { if (version < 3) { maxBytes = DEFAULT_RESPONSE_MAX_BYTES; } - return new FetchRequest(version, replicaId, maxWait, minBytes, maxBytes, fetchData, isolationLevel); + return new FetchRequest(version, replicaId, maxWait, minBytes, maxBytes, fetchData, + isolationLevel, toForget, metadata); } @Override @@ -257,13 +331,16 @@ public String toString() { append(", maxBytes=").append(maxBytes). append(", fetchData=").append(fetchData). append(", isolationLevel=").append(isolationLevel). + append(", toForget=").append(Utils.join(toForget, ", ")). + append(", metadata=").append(metadata). append(")"); return bld.toString(); } } private FetchRequest(short version, int replicaId, int maxWait, int minBytes, int maxBytes, - LinkedHashMap fetchData, IsolationLevel isolationLevel) { + Map fetchData, IsolationLevel isolationLevel, + List toForget, FetchMetadata metadata) { super(version); this.replicaId = replicaId; this.maxWait = maxWait; @@ -271,6 +348,8 @@ private FetchRequest(short version, int replicaId, int maxWait, int minBytes, in this.maxBytes = maxBytes; this.fetchData = fetchData; this.isolationLevel = isolationLevel; + this.toForget = toForget; + this.metadata = metadata; } public FetchRequest(Struct struct, short version) { @@ -282,11 +361,23 @@ public FetchRequest(Struct struct, short version) { maxBytes = struct.getInt(MAX_BYTES_KEY_NAME); else maxBytes = DEFAULT_RESPONSE_MAX_BYTES; - if (struct.hasField(ISOLATION_LEVEL_KEY_NAME)) isolationLevel = IsolationLevel.forId(struct.getByte(ISOLATION_LEVEL_KEY_NAME)); else isolationLevel = IsolationLevel.READ_UNCOMMITTED; + toForget = new ArrayList<>(0); + if (struct.hasField(FORGOTTEN_TOPICS_DATA)) { + for (Object forgottenTopicObj : struct.getArray(FORGOTTEN_TOPICS_DATA)) { + Struct forgottenTopic = (Struct) forgottenTopicObj; + String topicName = forgottenTopic.get(TOPIC_NAME); + for (Object partObj : forgottenTopic.getArray(PARTITIONS_KEY_NAME)) { + Integer part = (Integer) partObj; + toForget.add(new TopicPartition(topicName, part)); + } + } + } + metadata = new FetchMetadata(struct.getOrElse(SESSION_ID, INVALID_SESSION_ID), + struct.getOrElse(EPOCH, FINAL_EPOCH)); fetchData = new LinkedHashMap<>(); for (Object topicResponseObj : struct.getArray(TOPICS_KEY_NAME)) { @@ -307,15 +398,21 @@ public FetchRequest(Struct struct, short version) { @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { + // The error is indicated in two ways: by setting the same error code in all partitions, and by + // setting the top-level error code. The form where we set the same error code in all partitions + // is needed in order to maintain backwards compatibility with older versions of the protocol + // in which there was no top-level error code. Note that for incremental fetch responses, there + // may not be any partitions at all in the response. For this reason, the top-level error code + // is essential for them. + Errors error = Errors.forException(e); LinkedHashMap responseData = new LinkedHashMap<>(); - - for (Map.Entry entry: fetchData.entrySet()) { - FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData(Errors.forException(e), - FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, - null, MemoryRecords.EMPTY); + for (Map.Entry entry : fetchData.entrySet()) { + FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData(error, + FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, + FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY); responseData.put(entry.getKey(), partitionResponse); } - return new FetchResponse(responseData, throttleTimeMs); + return new FetchResponse(error, responseData, throttleTimeMs, metadata.sessionId()); } public int replicaId() { @@ -338,6 +435,10 @@ public Map fetchData() { return fetchData; } + public List toForget() { + return toForget; + } + public boolean isFromFollower() { return replicaId >= 0; } @@ -346,6 +447,10 @@ public IsolationLevel isolationLevel() { return isolationLevel; } + public FetchMetadata metadata() { + return metadata; + } + public static FetchRequest parse(ByteBuffer buffer, short version) { return new FetchRequest(ApiKeys.FETCH.parseRequest(version, buffer), version); } @@ -353,7 +458,8 @@ public static FetchRequest parse(ByteBuffer buffer, short version) { @Override protected Struct toStruct() { Struct struct = new Struct(ApiKeys.FETCH.requestSchema(version())); - List> topicsData = TopicAndPartitionData.batchByTopic(fetchData); + List> topicsData = + TopicAndPartitionData.batchByTopic(fetchData.entrySet().iterator()); struct.set(REPLICA_ID_KEY_NAME, replicaId); struct.set(MAX_WAIT_KEY_NAME, maxWait); @@ -362,6 +468,8 @@ protected Struct toStruct() { struct.set(MAX_BYTES_KEY_NAME, maxBytes); if (struct.hasField(ISOLATION_LEVEL_KEY_NAME)) struct.set(ISOLATION_LEVEL_KEY_NAME, isolationLevel.id()); + struct.setIfExists(SESSION_ID, metadata.sessionId()); + struct.setIfExists(EPOCH, metadata.epoch()); List topicArray = new ArrayList<>(); for (TopicAndPartitionData topicEntry : topicsData) { @@ -382,6 +490,25 @@ protected Struct toStruct() { topicArray.add(topicData); } struct.set(TOPICS_KEY_NAME, topicArray.toArray()); + if (struct.hasField(FORGOTTEN_TOPICS_DATA)) { + Map> topicsToPartitions = new HashMap<>(); + for (TopicPartition part : toForget) { + List partitions = topicsToPartitions.get(part.topic()); + if (partitions == null) { + partitions = new ArrayList<>(); + topicsToPartitions.put(part.topic(), partitions); + } + partitions.add(part.partition()); + } + List toForgetStructs = new ArrayList<>(); + for (Map.Entry> entry : topicsToPartitions.entrySet()) { + Struct toForgetStruct = struct.instance(FORGOTTEN_TOPICS_DATA); + toForgetStruct.set(TOPIC_NAME, entry.getKey()); + toForgetStruct.set(PARTITIONS_KEY_NAME, entry.getValue().toArray()); + toForgetStructs.add(toForgetStruct); + } + struct.set(FORGOTTEN_TOPICS_DATA, toForgetStructs.toArray()); + } return struct; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index 0d09027802ab8..98c6be333e15b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -31,6 +31,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -42,6 +43,7 @@ import static org.apache.kafka.common.protocol.types.Type.INT64; import static org.apache.kafka.common.protocol.types.Type.RECORDS; import static org.apache.kafka.common.protocol.types.Type.STRING; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; /** * This wrapper supports all versions of the Fetch API @@ -148,9 +150,19 @@ public class FetchResponse extends AbstractResponse { */ private static final Schema FETCH_RESPONSE_V6 = FETCH_RESPONSE_V5; + // FETCH_REESPONSE_V7 added incremental fetch responses and a top-level error code. + public static final Field.Int32 SESSION_ID = new Field.Int32("session_id", "The fetch session ID"); + + private static final Schema FETCH_RESPONSE_V7 = new Schema( + THROTTLE_TIME_MS, + ERROR_CODE, + SESSION_ID, + new Field(RESPONSES_KEY_NAME, new ArrayOf(FETCH_RESPONSE_TOPIC_V5))); + public static Schema[] schemaVersions() { return new Schema[] {FETCH_RESPONSE_V0, FETCH_RESPONSE_V1, FETCH_RESPONSE_V2, - FETCH_RESPONSE_V3, FETCH_RESPONSE_V4, FETCH_RESPONSE_V5, FETCH_RESPONSE_V6}; + FETCH_RESPONSE_V3, FETCH_RESPONSE_V4, FETCH_RESPONSE_V5, FETCH_RESPONSE_V6, + FETCH_RESPONSE_V7}; } @@ -168,8 +180,10 @@ public static Schema[] schemaVersions() { * UNKNOWN (-1) */ - private final LinkedHashMap responseData; private final int throttleTimeMs; + private final Errors error; + private final int sessionId; + private final LinkedHashMap responseData; public static final class AbortedTransaction { public final long producerId; @@ -268,17 +282,20 @@ public String toString() { } /** - * Constructor for all versions. - * * From version 3 or later, the entries in `responseData` should be in the same order as the entries in * `FetchRequest.fetchData`. * - * @param responseData fetched data grouped by topic-partition - * @param throttleTimeMs Time in milliseconds the response was throttled + * @param error The top-level error code. + * @param responseData The fetched data grouped by partition. + * @param throttleTimeMs The time in milliseconds that the response was throttled + * @param sessionId The fetch session id. */ - public FetchResponse(LinkedHashMap responseData, int throttleTimeMs) { + public FetchResponse(Errors error, LinkedHashMap responseData, + int throttleTimeMs, int sessionId) { + this.error = error; this.responseData = responseData; this.throttleTimeMs = throttleTimeMs; + this.sessionId = sessionId; } public FetchResponse(Struct struct) { @@ -316,17 +333,19 @@ public FetchResponse(Struct struct) { } PartitionData partitionData = new PartitionData(error, highWatermark, lastStableOffset, logStartOffset, - abortedTransactions, records); + abortedTransactions, records); responseData.put(new TopicPartition(topic, partition), partitionData); } } this.responseData = responseData; this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME); + this.error = Errors.forCode(struct.getOrElse(ERROR_CODE, (short) 0)); + this.sessionId = struct.getOrElse(SESSION_ID, INVALID_SESSION_ID); } @Override public Struct toStruct(short version) { - return toStruct(version, responseData, throttleTimeMs); + return toStruct(version, throttleTimeMs, error, responseData.entrySet().iterator(), sessionId); } @Override @@ -346,6 +365,10 @@ protected Send toSend(String dest, ResponseHeader responseHeader, short apiVersi return new MultiSend(dest, sends); } + public Errors error() { + return error; + } + public LinkedHashMap responseData() { return responseData; } @@ -354,6 +377,10 @@ public int throttleTimeMs() { return this.throttleTimeMs; } + public int sessionId() { + return sessionId; + } + @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); @@ -369,7 +396,15 @@ public static FetchResponse parse(ByteBuffer buffer, short version) { private static void addResponseData(Struct struct, int throttleTimeMs, String dest, List sends) { Object[] allTopicData = struct.getArray(RESPONSES_KEY_NAME); - if (struct.hasField(THROTTLE_TIME_MS)) { + if (struct.hasField(ERROR_CODE)) { + ByteBuffer buffer = ByteBuffer.allocate(14); + buffer.putInt(throttleTimeMs); + buffer.putShort(struct.get(ERROR_CODE)); + buffer.putInt(struct.get(SESSION_ID)); + buffer.putInt(allTopicData.length); + buffer.rewind(); + sends.add(new ByteBufferSend(dest, buffer)); + } else if (struct.hasField(THROTTLE_TIME_MS)) { ByteBuffer buffer = ByteBuffer.allocate(8); buffer.putInt(throttleTimeMs); buffer.putInt(allTopicData.length); @@ -416,9 +451,14 @@ private static void addPartitionData(String dest, List sends, Struct parti sends.add(new RecordsSend(dest, records)); } - private static Struct toStruct(short version, LinkedHashMap responseData, int throttleTimeMs) { + private static Struct toStruct(short version, int throttleTimeMs, Errors error, + Iterator> partIterator, int sessionId) { Struct struct = new Struct(ApiKeys.FETCH.responseSchema(version)); - List> topicsData = FetchRequest.TopicAndPartitionData.batchByTopic(responseData); + struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs); + struct.setIfExists(ERROR_CODE, error.code()); + struct.setIfExists(SESSION_ID, sessionId); + List> topicsData = + FetchRequest.TopicAndPartitionData.batchByTopic(partIterator); List topicArray = new ArrayList<>(); for (FetchRequest.TopicAndPartitionData topicEntry: topicsData) { Struct topicData = struct.instance(RESPONSES_KEY_NAME); @@ -466,13 +506,20 @@ private static Struct toStruct(short version, LinkedHashMap responseData) { - return 4 + toStruct(version, responseData, 0).sizeOf(); + /** + * Convenience method to find the size of a response. + * + * @param version The version of the response to use. + * @param partIterator The partition iterator. + * @return The response size in bytes. + */ + public static int sizeOf(short version, Iterator> partIterator) { + // Since the throttleTimeMs and metadata field sizes are constant and fixed, we can + // use arbitrary values here without affecting the result. + return 4 + toStruct(version, 0, Errors.NONE, partIterator, INVALID_SESSION_ID).sizeOf(); } } diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java new file mode 100644 index 0000000000000..701684dd0b451 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.utils; + +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A LinkedHashSet which is more memory-efficient than the standard implementation. + * + * This set preserves the order of insertion. The order of iteration will always be + * the order of insertion. + * + * This collection requires previous and next indexes to be embedded into each + * element. Using array indices rather than pointers saves space on large heaps + * where pointer compression is not in use. It also reduces the amount of time + * the garbage collector has to spend chasing pointers. + * + * This class uses linear probing. Unlike HashMap (but like HashTable), we don't force + * the size to be a power of 2. This saves memory. + * + * This class does not have internal synchronization. + */ +@SuppressWarnings("unchecked") +public class ImplicitLinkedHashSet extends AbstractSet { + public interface Element { + int prev(); + void setPrev(int e); + int next(); + void setNext(int e); + } + + private static final int HEAD_INDEX = -1; + + public static final int INVALID_INDEX = -2; + + private static class HeadElement implements Element { + private int prev = HEAD_INDEX; + private int next = HEAD_INDEX; + + @Override + public int prev() { + return prev; + } + + @Override + public void setPrev(int prev) { + this.prev = prev; + } + + @Override + public int next() { + return next; + } + + @Override + public void setNext(int next) { + this.next = next; + } + } + + private static Element indexToElement(Element head, Element[] elements, int index) { + if (index == HEAD_INDEX) { + return head; + } + return elements[index]; + } + + private static void addToListTail(Element head, Element[] elements, int elementIdx) { + int oldTailIdx = head.prev(); + Element element = indexToElement(head, elements, elementIdx); + Element oldTail = indexToElement(head, elements, oldTailIdx); + head.setPrev(elementIdx); + oldTail.setNext(elementIdx); + element.setPrev(oldTailIdx); + element.setNext(HEAD_INDEX); + } + + private static void removeFromList(Element head, Element[] elements, int elementIdx) { + Element element = indexToElement(head, elements, elementIdx); + elements[elementIdx] = null; + int prevIdx = element.prev(); + int nextIdx = element.next(); + Element prev = indexToElement(head, elements, prevIdx); + Element next = indexToElement(head, elements, nextIdx); + prev.setNext(nextIdx); + next.setPrev(prevIdx); + element.setNext(INVALID_INDEX); + element.setPrev(INVALID_INDEX); + } + + private class ImplicitLinkedHashSetIterator implements Iterator { + private Element cur = head; + + private Element next = indexToElement(head, elements, head.next()); + + @Override + public boolean hasNext() { + return next != head; + } + + @Override + public E next() { + if (next == head) { + throw new NoSuchElementException(); + } + cur = next; + next = indexToElement(head, elements, cur.next()); + return (E) cur; + } + + @Override + public void remove() { + if (cur == head) { + throw new IllegalStateException(); + } + ImplicitLinkedHashSet.this.remove(cur); + cur = head; + } + } + + private Element head; + + private Element[] elements; + + private int size; + + @Override + public Iterator iterator() { + return new ImplicitLinkedHashSetIterator(); + } + + private static int slot(Element[] curElements, Element e) { + return (e.hashCode() & 0x7fffffff) % curElements.length; + } + + /** + * Find an element matching an example element. + * + * Using the element's hash code, we can look up the slot where it belongs. + * However, it may not have ended up in exactly this slot, due to a collision. + * Therefore, we must search forward in the array until we hit a null, before + * concluding that the element is not present. + * + * @param example The element to match. + * @return The match index, or INVALID_INDEX if no match was found. + */ + private int findIndex(E example) { + int slot = slot(elements, example); + for (int seen = 0; seen < elements.length; seen++) { + Element element = elements[slot]; + if (element == null) { + return INVALID_INDEX; + } + if (element.equals(example)) { + return slot; + } + slot = (slot + 1) % elements.length; + } + return INVALID_INDEX; + } + + /** + * Find the element which equals() the given example element. + * + * @param example The example element. + * @return Null if no element was found; the element, otherwise. + */ + public E find(E example) { + int index = findIndex(example); + if (index == INVALID_INDEX) { + return null; + } + return (E) elements[index]; + } + + /** + * Returns the number of elements in the set. + */ + @Override + public int size() { + return size; + } + + @Override + public boolean contains(Object o) { + E example = null; + try { + example = (E) o; + } catch (ClassCastException e) { + return false; + } + return find(example) != null; + } + + @Override + public boolean add(E newElement) { + if ((size + 1) >= elements.length / 2) { + // Avoid using even-sized capacities, to get better key distribution. + changeCapacity((2 * elements.length) + 1); + } + int slot = addInternal(newElement, elements); + if (slot >= 0) { + addToListTail(head, elements, slot); + size++; + return true; + } + return false; + } + + public void mustAdd(E newElement) { + if (!add(newElement)) { + throw new RuntimeException("Unable to add " + newElement); + } + } + + /** + * Adds a new element to the appropriate place in the elements array. + * + * @param newElement The new element to add. + * @param addElements The elements array. + * @return The index at which the element was inserted, or INVALID_INDEX + * if the element could not be inserted because there was already + * an equivalent element. + */ + private static int addInternal(Element newElement, Element[] addElements) { + int slot = slot(addElements, newElement); + for (int seen = 0; seen < addElements.length; seen++) { + Element element = addElements[slot]; + if (element == null) { + addElements[slot] = newElement; + return slot; + } + if (element.equals(newElement)) { + return INVALID_INDEX; + } + slot = (slot + 1) % addElements.length; + } + throw new RuntimeException("Not enough hash table slots to add a new element."); + } + + private void changeCapacity(int newCapacity) { + Element[] newElements = new Element[newCapacity]; + HeadElement newHead = new HeadElement(); + int oldSize = size; + for (Iterator iter = iterator(); iter.hasNext(); ) { + Element element = iter.next(); + iter.remove(); + int newSlot = addInternal(element, newElements); + addToListTail(newHead, newElements, newSlot); + } + this.elements = newElements; + this.head = newHead; + this.size = oldSize; + } + + @Override + public boolean remove(Object o) { + E example = null; + try { + example = (E) o; + } catch (ClassCastException e) { + return false; + } + int slot = findIndex(example); + if (slot == INVALID_INDEX) { + return false; + } + size--; + removeFromList(head, elements, slot); + slot = (slot + 1) % elements.length; + + // Find the next empty slot + int endSlot = slot; + for (int seen = 0; seen < elements.length; seen++) { + Element element = elements[endSlot]; + if (element == null) { + break; + } + endSlot = (endSlot + 1) % elements.length; + } + + // We must preserve the denseness invariant. The denseness invariant says that + // any element is either in the slot indicated by its hash code, or a slot which + // is not separated from that slot by any nulls. + // Reseat all elements in between the deleted element and the next empty slot. + while (slot != endSlot) { + reseat(slot); + slot = (slot + 1) % elements.length; + } + return true; + } + + private void reseat(int prevSlot) { + Element element = elements[prevSlot]; + int newSlot = slot(elements, element); + for (int seen = 0; seen < elements.length; seen++) { + Element e = elements[newSlot]; + if ((e == null) || (e == element)) { + break; + } + newSlot = (newSlot + 1) % elements.length; + } + if (newSlot == prevSlot) { + return; + } + Element prev = indexToElement(head, elements, element.prev()); + prev.setNext(newSlot); + Element next = indexToElement(head, elements, element.next()); + next.setPrev(newSlot); + elements[prevSlot] = null; + elements[newSlot] = element; + } + + @Override + public void clear() { + reset(elements.length); + } + + public ImplicitLinkedHashSet() { + this(5); + } + + public ImplicitLinkedHashSet(int initialCapacity) { + reset(initialCapacity); + } + + private void reset(int capacity) { + this.head = new HeadElement(); + // Avoid using even-sized capacities, to get better key distribution. + this.elements = new Element[(2 * capacity) + 1]; + this.size = 0; + } + + int numSlots() { + return elements.length; + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java new file mode 100644 index 0000000000000..3095717e8dd72 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.FetchRequest; +import org.apache.kafka.common.requests.FetchResponse; +import org.apache.kafka.common.utils.LogContext; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import static org.apache.kafka.common.requests.FetchMetadata.INITIAL_EPOCH; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * A unit test for FetchSessionHandler. + */ +public class FetchSessionHandlerTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + private static final LogContext LOG_CONTEXT = new LogContext("[FetchSessionHandler]="); + + private static final Logger log = LOG_CONTEXT.logger(FetchSessionHandler.class); + + /** + * Create a set of TopicPartitions. We use a TreeSet, in order to get a deterministic + * ordering for test purposes. + */ + private final static Set toSet(TopicPartition... arr) { + TreeSet set = new TreeSet<>(new Comparator() { + @Override + public int compare(TopicPartition o1, TopicPartition o2) { + return o1.toString().compareTo(o2.toString()); + } + }); + set.addAll(Arrays.asList(arr)); + return set; + } + + @Test + public void testFindMissing() throws Exception { + TopicPartition foo0 = new TopicPartition("foo", 0); + TopicPartition foo1 = new TopicPartition("foo", 1); + TopicPartition bar0 = new TopicPartition("bar", 0); + TopicPartition bar1 = new TopicPartition("bar", 1); + TopicPartition baz0 = new TopicPartition("baz", 0); + TopicPartition baz1 = new TopicPartition("baz", 1); + assertEquals(toSet(), FetchSessionHandler.findMissing(toSet(foo0), toSet(foo0))); + assertEquals(toSet(foo0), FetchSessionHandler.findMissing(toSet(foo0), toSet(foo1))); + assertEquals(toSet(foo0, foo1), + FetchSessionHandler.findMissing(toSet(foo0, foo1), toSet(baz0))); + assertEquals(toSet(bar1, foo0, foo1), + FetchSessionHandler.findMissing(toSet(foo0, foo1, bar0, bar1), + toSet(bar0, baz0, baz1))); + assertEquals(toSet(), + FetchSessionHandler.findMissing(toSet(foo0, foo1, bar0, bar1, baz1), + toSet(foo0, foo1, bar0, bar1, baz0, baz1))); + } + + private static final class ReqEntry { + final TopicPartition part; + final FetchRequest.PartitionData data; + + ReqEntry(String topic, int partition, long fetchOffset, long logStartOffset, int maxBytes) { + this.part = new TopicPartition(topic, partition); + this.data = new FetchRequest.PartitionData(fetchOffset, logStartOffset, maxBytes); + } + } + + private static LinkedHashMap reqMap(ReqEntry... entries) { + LinkedHashMap map = new LinkedHashMap<>(); + for (ReqEntry entry : entries) { + map.put(entry.part, entry.data); + } + return map; + } + + private static void assertMapEquals(Map expected, + Map actual) { + Iterator> expectedIter = + expected.entrySet().iterator(); + Iterator> actualIter = + actual.entrySet().iterator(); + int i = 1; + while (expectedIter.hasNext()) { + Map.Entry expectedEntry = expectedIter.next(); + if (!actualIter.hasNext()) { + fail("Element " + i + " not found."); + } + Map.Entry actuaLEntry = actualIter.next(); + assertEquals("Element " + i + " had a different TopicPartition than expected.", + expectedEntry.getKey(), actuaLEntry.getKey()); + assertEquals("Element " + i + " had different PartitionData than expected.", + expectedEntry.getValue(), actuaLEntry.getValue()); + i++; + } + if (expectedIter.hasNext()) { + fail("Unexpected element " + i + " found."); + } + } + + private static void assertMapsEqual(Map expected, + Map... actuals) { + for (Map actual : actuals) { + assertMapEquals(expected, actual); + } + } + + private static void assertListEquals(List expected, List actual) { + for (TopicPartition expectedPart : expected) { + if (!actual.contains(expectedPart)) { + fail("Failed to find expected partition " + expectedPart); + } + } + for (TopicPartition actualPart : actual) { + if (!expected.contains(actualPart)) { + fail("Found unexpected partition " + actualPart); + } + } + } + + private static final class RespEntry { + final TopicPartition part; + final FetchResponse.PartitionData data; + + RespEntry(String topic, int partition, long highWatermark, long lastStableOffset) { + this.part = new TopicPartition(topic, partition); + this.data = new FetchResponse.PartitionData( + Errors.NONE, + highWatermark, + lastStableOffset, + 0, + null, + null); + } + } + + private static LinkedHashMap respMap(RespEntry... entries) { + LinkedHashMap map = new LinkedHashMap<>(); + for (RespEntry entry : entries) { + map.put(entry.part, entry.data); + } + return map; + } + + /** + * Test the handling of SESSIONLESS responses. + * Pre-KIP-227 brokers always supply this kind of response. + */ + @Test + public void testSessionless() throws Exception { + FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1); + FetchSessionHandler.Builder builder = handler.newBuilder(); + builder.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + builder.add(new TopicPartition("foo", 1), + new FetchRequest.PartitionData(10, 110, 210)); + FetchSessionHandler.FetchRequestData data = builder.build(); + assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200), + new ReqEntry("foo", 1, 10, 110, 210)), + data.toSend(), data.sessionPartitions()); + assertEquals(INVALID_SESSION_ID, data.metadata().sessionId()); + assertEquals(INITIAL_EPOCH, data.metadata().epoch()); + + FetchResponse resp = new FetchResponse(Errors.NONE, + respMap(new RespEntry("foo", 0, 0, 0), + new RespEntry("foo", 1, 0, 0)), + 0, INVALID_SESSION_ID); + handler.handleResponse(resp); + + FetchSessionHandler.Builder builder2 = handler.newBuilder(); + builder2.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + FetchSessionHandler.FetchRequestData data2 = builder2.build(); + assertEquals(INVALID_SESSION_ID, data2.metadata().sessionId()); + assertEquals(INITIAL_EPOCH, data2.metadata().epoch()); + assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200)), + data.toSend(), data.sessionPartitions()); + } + + /** + * Test handling an incremental fetch session. + */ + @Test + public void testIncrementals() throws Exception { + FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1); + FetchSessionHandler.Builder builder = handler.newBuilder(); + builder.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + builder.add(new TopicPartition("foo", 1), + new FetchRequest.PartitionData(10, 110, 210)); + FetchSessionHandler.FetchRequestData data = builder.build(); + assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200), + new ReqEntry("foo", 1, 10, 110, 210)), + data.toSend(), data.sessionPartitions()); + assertEquals(INVALID_SESSION_ID, data.metadata().sessionId()); + assertEquals(INITIAL_EPOCH, data.metadata().epoch()); + + FetchResponse resp = new FetchResponse(Errors.NONE, + respMap(new RespEntry("foo", 0, 10, 20), + new RespEntry("foo", 1, 10, 20)), + 0, 123); + handler.handleResponse(resp); + + // Test an incremental fetch request which adds one partition and modifies another. + FetchSessionHandler.Builder builder2 = handler.newBuilder(); + builder2.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + builder2.add(new TopicPartition("foo", 1), + new FetchRequest.PartitionData(10, 120, 210)); + builder2.add(new TopicPartition("bar", 0), + new FetchRequest.PartitionData(20, 200, 200)); + FetchSessionHandler.FetchRequestData data2 = builder2.build(); + assertFalse(data2.metadata().isFull()); + assertMapEquals(reqMap(new ReqEntry("foo", 0, 0, 100, 200), + new ReqEntry("foo", 1, 10, 120, 210), + new ReqEntry("bar", 0, 20, 200, 200)), + data2.sessionPartitions()); + assertMapEquals(reqMap(new ReqEntry("bar", 0, 20, 200, 200), + new ReqEntry("foo", 1, 10, 120, 210)), + data2.toSend()); + + FetchResponse resp2 = new FetchResponse(Errors.NONE, + respMap(new RespEntry("foo", 1, 20, 20)), + 0, 123); + handler.handleResponse(resp2); + + // Skip building a new request. Test that handling an invalid fetch session epoch response results + // in a request which closes the session. + FetchResponse resp3 = new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, respMap(), + 0, INVALID_SESSION_ID); + handler.handleResponse(resp3); + + FetchSessionHandler.Builder builder4 = handler.newBuilder(); + builder4.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + builder4.add(new TopicPartition("foo", 1), + new FetchRequest.PartitionData(10, 120, 210)); + builder4.add(new TopicPartition("bar", 0), + new FetchRequest.PartitionData(20, 200, 200)); + FetchSessionHandler.FetchRequestData data4 = builder4.build(); + assertTrue(data4.metadata().isFull()); + assertEquals(data2.metadata().sessionId(), data4.metadata().sessionId()); + assertEquals(INITIAL_EPOCH, data4.metadata().epoch()); + assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200), + new ReqEntry("foo", 1, 10, 120, 210), + new ReqEntry("bar", 0, 20, 200, 200)), + data4.sessionPartitions(), data4.toSend()); + } + + /** + * Test that calling FetchSessionHandler#Builder#build twice fails. + */ + @Test + public void testDoubleBuild() throws Exception { + FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1); + FetchSessionHandler.Builder builder = handler.newBuilder(); + builder.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + builder.build(); + try { + builder.build(); + fail("Expected calling build twice to fail."); + } catch (Throwable t) { + // expected + } + } + + @Test + public void testIncrementalPartitionRemoval() throws Exception { + FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1); + FetchSessionHandler.Builder builder = handler.newBuilder(); + builder.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + builder.add(new TopicPartition("foo", 1), + new FetchRequest.PartitionData(10, 110, 210)); + builder.add(new TopicPartition("bar", 0), + new FetchRequest.PartitionData(20, 120, 220)); + FetchSessionHandler.FetchRequestData data = builder.build(); + assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200), + new ReqEntry("foo", 1, 10, 110, 210), + new ReqEntry("bar", 0, 20, 120, 220)), + data.toSend(), data.sessionPartitions()); + assertTrue(data.metadata().isFull()); + + FetchResponse resp = new FetchResponse(Errors.NONE, + respMap(new RespEntry("foo", 0, 10, 20), + new RespEntry("foo", 1, 10, 20), + new RespEntry("bar", 0, 10, 20)), + 0, 123); + handler.handleResponse(resp); + + // Test an incremental fetch request which removes two partitions. + FetchSessionHandler.Builder builder2 = handler.newBuilder(); + builder2.add(new TopicPartition("foo", 1), + new FetchRequest.PartitionData(10, 110, 210)); + FetchSessionHandler.FetchRequestData data2 = builder2.build(); + assertFalse(data2.metadata().isFull()); + assertEquals(123, data2.metadata().sessionId()); + assertEquals(1, data2.metadata().epoch()); + assertMapEquals(reqMap(new ReqEntry("foo", 1, 10, 110, 210)), + data2.sessionPartitions()); + assertMapEquals(reqMap(), data2.toSend()); + ArrayList expectedToForget2 = new ArrayList<>(); + expectedToForget2.add(new TopicPartition("foo", 0)); + expectedToForget2.add(new TopicPartition("bar", 0)); + assertListEquals(expectedToForget2, data2.toForget()); + + // A FETCH_SESSION_ID_NOT_FOUND response triggers us to close the session. + // The next request is a session establishing FULL request. + FetchResponse resp2 = new FetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, + respMap(), 0, INVALID_SESSION_ID); + handler.handleResponse(resp2); + FetchSessionHandler.Builder builder3 = handler.newBuilder(); + builder3.add(new TopicPartition("foo", 0), + new FetchRequest.PartitionData(0, 100, 200)); + FetchSessionHandler.FetchRequestData data3 = builder3.build(); + assertTrue(data3.metadata().isFull()); + assertEquals(INVALID_SESSION_ID, data3.metadata().sessionId()); + assertEquals(INITIAL_EPOCH, data3.metadata().epoch()); + assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200)), + data3.sessionPartitions(), data3.toSend()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index a827168039ecb..d47124f8f3e56 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -99,6 +99,7 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -1578,7 +1579,7 @@ private FetchResponse fetchResponse(Map fetches) { tpResponses.put(partition, new FetchResponse.PartitionData(Errors.NONE, 0, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); } - return new FetchResponse(tpResponses, 0); + return new FetchResponse(Errors.NONE, tpResponses, 0, INVALID_SESSION_ID); } private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, int count) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index a3ea79356f94e..a0205e7f19ec2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -103,6 +103,7 @@ import java.util.Set; import static java.util.Collections.singleton; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -139,6 +140,7 @@ public class FetcherTest { private MemoryRecords records; private MemoryRecords nextRecords; + private MemoryRecords emptyRecords; private Fetcher fetcher = createFetcher(subscriptions, metrics); private Metrics fetcherMetrics = new Metrics(time); private Fetcher fetcherNoAutoReset = createFetcher(subscriptionsNoAutoReset, fetcherMetrics); @@ -158,6 +160,9 @@ public void setup() throws Exception { builder.append(0L, "key".getBytes(), "value-4".getBytes()); builder.append(0L, "key".getBytes(), "value-5".getBytes()); nextRecords = builder.build(); + + builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L); + emptyRecords = builder.build(); } @After @@ -177,7 +182,7 @@ public void testFetchNormal() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -219,7 +224,7 @@ public void testFetcherIgnoresControlRecords() { buffer.flip(); - client.prepareResponse(fetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -242,7 +247,7 @@ public void testFetchError() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -283,7 +288,7 @@ public byte[] deserialize(String topic, byte[] data) { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tp0, 1), fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tp0, 1), fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(0); @@ -345,7 +350,7 @@ public void testParseCorruptedRecord() throws Exception { // normal fetch assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(0); // the first fetchedRecords() should return the first valid message @@ -383,7 +388,7 @@ private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { // Should not throw exception after the seek. fetcher.fetchedRecords(); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); consumerClient.poll(0); List> records = fetcher.fetchedRecords().get(tp0); @@ -416,7 +421,7 @@ public void testInvalidDefaultRecordBatch() { // normal fetch assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(0); // the fetchedRecords() should always throw exception due to the bad batch. @@ -447,7 +452,7 @@ public void testParseInvalidRecordBatch() throws Exception { // normal fetch assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(0); try { fetcher.fetchedRecords(); @@ -480,7 +485,7 @@ public void testHeaders() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tp0, 1), fetchResponse(tp0, memoryRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tp0, 1), fullFetchResponse(tp0, memoryRecords, Errors.NONE, 100L, 0)); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(0); @@ -510,8 +515,8 @@ public void testFetchMaxPollRecords() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tp0, 1), fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); - client.prepareResponse(matchesOffset(tp0, 4), fetchResponse(tp0, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tp0, 1), fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tp0, 4), fullFetchResponse(tp0, this.nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(0); @@ -551,7 +556,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { subscriptions.seek(tp0, 1); // Returns 3 records while `max.poll.records` is configured to 2 - client.prepareResponse(matchesOffset(tp0, 1), fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tp0, 1), fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(0); @@ -562,7 +567,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(2, records.get(1).offset()); subscriptions.assignFromUser(singleton(tp1)); - client.prepareResponse(matchesOffset(tp1, 4), fetchResponse(tp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tp1, 4), fullFetchResponse(tp1, this.nextRecords, Errors.NONE, 100L, 0)); subscriptions.seek(tp1, 4); assertEquals(1, fetcher.sendFetches()); @@ -594,7 +599,7 @@ public void testFetchNonContinuousRecords() { // normal fetch assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(0); consumerRecords = fetcher.fetchedRecords().get(tp0); assertEquals(3, consumerRecords.size()); @@ -654,7 +659,7 @@ private void makeFetchRequestWithIncompleteRecord() { assertFalse(fetcher.hasCompletedFetches()); MemoryRecords partialRecord = MemoryRecords.readableRecords( ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0})); - client.prepareResponse(fetchResponse(tp0, partialRecord, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, partialRecord, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); } @@ -666,7 +671,7 @@ public void testUnauthorizedTopic() { // resize the limit of the buffer to pretend it is only fetch-size large assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); consumerClient.poll(0); try { fetcher.fetchedRecords(); @@ -686,7 +691,7 @@ public void testFetchDuringRebalance() { // Now the rebalance happens and fetch positions are cleared subscriptions.assignFromSubscribed(singleton(tp0)); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(0); // The active fetch should be ignored since its position is no longer valid @@ -701,7 +706,7 @@ public void testInFlightFetchOnPausedPartition() { assertEquals(1, fetcher.sendFetches()); subscriptions.pause(tp0); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertNull(fetcher.fetchedRecords().get(tp0)); } @@ -722,7 +727,7 @@ public void testFetchNotLeaderForPartition() { subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); consumerClient.poll(0); assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -734,7 +739,7 @@ public void testFetchUnknownTopicOrPartition() { subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); consumerClient.poll(0); assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -746,7 +751,7 @@ public void testFetchOffsetOutOfRange() { subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(0); assertEquals(0, fetcher.fetchedRecords().size()); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); @@ -761,7 +766,7 @@ public void testStaleOutOfRangeError() { subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); subscriptions.seek(tp0, 1); consumerClient.poll(0); assertEquals(0, fetcher.fetchedRecords().size()); @@ -775,7 +780,7 @@ public void testFetchedRecordsAfterSeek() { subscriptionsNoAutoReset.seek(tp0, 0); assertTrue(fetcherNoAutoReset.sendFetches() > 0); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(0); assertFalse(subscriptionsNoAutoReset.isOffsetResetNeeded(tp0)); subscriptionsNoAutoReset.seek(tp0, 2); @@ -788,7 +793,7 @@ public void testFetchOffsetOutOfRangeException() { subscriptionsNoAutoReset.seek(tp0, 0); fetcherNoAutoReset.sendFetches(); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(0); assertFalse(subscriptionsNoAutoReset.isOffsetResetNeeded(tp0)); @@ -818,7 +823,8 @@ public void testFetchPositionAfterException() { FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, records)); partitions.put(tp0, new FetchResponse.PartitionData(Errors.OFFSET_OUT_OF_RANGE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); - client.prepareResponse(new FetchResponse(new LinkedHashMap<>(partitions), 0)); + client.prepareResponse(new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), + 0, INVALID_SESSION_ID)); consumerClient.poll(0); List> fetchedRecords = new ArrayList<>(); @@ -856,7 +862,7 @@ public void testSeekBeforeException() { Map partitions = new HashMap<>(); partitions.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, records)); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertEquals(2, fetcher.fetchedRecords().get(tp0).size()); @@ -867,7 +873,7 @@ public void testSeekBeforeException() { partitions = new HashMap<>(); partitions.put(tp1, new FetchResponse.PartitionData(Errors.OFFSET_OUT_OF_RANGE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); - client.prepareResponse(new FetchResponse(new LinkedHashMap<>(partitions), 0)); + client.prepareResponse(new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), 0, INVALID_SESSION_ID)); consumerClient.poll(0); assertEquals(1, fetcher.fetchedRecords().get(tp0).size()); @@ -882,7 +888,7 @@ public void testFetchDisconnected() { subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0), true); consumerClient.poll(0); assertEquals(0, fetcher.fetchedRecords().size()); @@ -1148,7 +1154,7 @@ public void testQuotaMetrics() throws Exception { ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true, null); client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); - FetchResponse response = fetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); + FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); buffer = response.serialize(ApiKeys.FETCH.latestVersion(), new ResponseHeader(request.correlationId())); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -1325,7 +1331,8 @@ public void testFetchResponseMetricsWithOnePartitionError() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, MemoryRecords.EMPTY)); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(new FetchResponse(new LinkedHashMap<>(partitions), 0)); + client.prepareResponse(new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), + 0, INVALID_SESSION_ID)); consumerClient.poll(0); fetcher.fetchedRecords(); @@ -1364,7 +1371,8 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); - client.prepareResponse(new FetchResponse(new LinkedHashMap<>(partitions), 0)); + client.prepareResponse(new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), + 0, INVALID_SESSION_ID)); consumerClient.poll(0); fetcher.fetchedRecords(); @@ -1390,7 +1398,7 @@ public void testFetcherMetricsTemplates() throws Exception { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetcher.fetchedRecords(); @@ -1417,7 +1425,7 @@ private Map>> fetchRecords( private Map>> fetchRecords( TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); + client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); consumerClient.poll(0); return fetcher.fetchedRecords(); } @@ -1495,7 +1503,7 @@ public void testSkippingAbortedTransactions() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1533,7 +1541,7 @@ public boolean matches(AbstractRequest body) { assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); return true; } - }, fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1604,7 +1612,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1651,7 +1659,7 @@ public void testMultipleAbortMarkers() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1695,7 +1703,7 @@ public void testReadCommittedAbortMarkerWithNoData() { List abortedTransactions = new ArrayList<>(); abortedTransactions.add(new FetchResponse.AbortedTransaction(producerId, 0L)); - client.prepareResponse(fetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), + client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1733,7 +1741,7 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, compactedRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, compactedRecords, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1768,7 +1776,7 @@ public void testUpdatePositionOnEmptyBatch() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fetchResponse(tp0, recordsWithEmptyBatch, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, recordsWithEmptyBatch, Errors.NONE, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1829,7 +1837,7 @@ public void testReadCommittedWithCompactedTopic() { abortedTransactions.add(new FetchResponse.AbortedTransaction(pid2, 6L)); abortedTransactions.add(new FetchResponse.AbortedTransaction(pid1, 0L)); - client.prepareResponse(fetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), + client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1867,7 +1875,7 @@ public void testReturnAbortedTransactionsinUncommittedMode() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1900,7 +1908,7 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(0); assertTrue(fetcher.hasCompletedFetches()); @@ -1911,6 +1919,75 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { assertEquals(currentOffset, (long) subscriptions.position(tp0)); } + @Test + public void testConsumingViaIncrementalFetchRequests() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(time), 2); + + List> records; + subscriptions.assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1))); + subscriptions.seek(tp0, 0); + subscriptions.seek(tp1, 1); + + // Fetch some records and establish an incremental fetch session. + LinkedHashMap partitions1 = new LinkedHashMap<>(); + partitions1.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 2L, + 2, 0L, null, this.records)); + partitions1.put(tp1, new FetchResponse.PartitionData(Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, emptyRecords)); + FetchResponse resp1 = new FetchResponse(Errors.NONE, partitions1, 0, 123); + client.prepareResponse(resp1); + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + Map>> fetchedRecords = fetcher.fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).longValue()); + assertEquals(1L, subscriptions.position(tp1).longValue()); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + // There is still a buffered record. + assertEquals(0, fetcher.sendFetches()); + fetchedRecords = fetcher.fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(1, records.size()); + assertEquals(3, records.get(0).offset()); + assertEquals(4L, subscriptions.position(tp0).longValue()); + + // The second response contains no new records. + LinkedHashMap partitions2 = new LinkedHashMap<>(); + FetchResponse resp2 = new FetchResponse(Errors.NONE, partitions2, 0, 123); + client.prepareResponse(resp2); + assertEquals(1, fetcher.sendFetches()); + consumerClient.poll(0); + fetchedRecords = fetcher.fetchedRecords(); + assertTrue(fetchedRecords.isEmpty()); + assertEquals(4L, subscriptions.position(tp0).longValue()); + assertEquals(1L, subscriptions.position(tp1).longValue()); + + // The third response contains some new records for tp0. + LinkedHashMap partitions3 = new LinkedHashMap<>(); + partitions3.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 100L, + 4, 0L, null, this.nextRecords)); + new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions1), 0, INVALID_SESSION_ID); + FetchResponse resp3 = new FetchResponse(Errors.NONE, partitions3, 0, 123); + client.prepareResponse(resp3); + assertEquals(1, fetcher.sendFetches()); + consumerClient.poll(0); + fetchedRecords = fetcher.fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp0).longValue()); + assertEquals(1L, subscriptions.position(tp1).longValue()); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, int baseSequence, SimpleRecord... records) { MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseSequence, true, @@ -2033,7 +2110,7 @@ private ListOffsetResponse listOffsetResponse(TopicPartition tp, Errors error, l return new ListOffsetResponse(allPartitionData); } - private FetchResponse fetchResponseWithAbortedTransactions(MemoryRecords records, + private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, List abortedTransactions, Errors error, long lastStableOffset, @@ -2041,18 +2118,18 @@ private FetchResponse fetchResponseWithAbortedTransactions(MemoryRecords records int throttleTime) { Map partitions = Collections.singletonMap(tp0, new FetchResponse.PartitionData(error, hw, lastStableOffset, 0L, abortedTransactions, records)); - return new FetchResponse(new LinkedHashMap<>(partitions), throttleTime); + return new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); } - private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { - return fetchResponse(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fullFetchResponse(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); } - private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { Map partitions = Collections.singletonMap(tp, new FetchResponse.PartitionData(error, hw, lastStableOffset, 0L, null, records)); - return new FetchResponse(new LinkedHashMap<>(partitions), throttleTime); + return new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); } private MetadataResponse newMetadataResponse(String topic, Errors error) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 0f7429e67e05d..bdbd10626856a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -75,6 +75,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; import static org.apache.kafka.test.TestUtils.toBuffer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -97,6 +98,13 @@ public void testSerialization() throws Exception { checkErrorResponse(createControlledShutdownRequest(0), new UnknownServerException()); checkRequest(createFetchRequest(4)); checkResponse(createFetchResponse(), 4); + List toForgetTopics = new ArrayList<>(); + toForgetTopics.add(new TopicPartition("foo", 0)); + toForgetTopics.add(new TopicPartition("foo", 2)); + toForgetTopics.add(new TopicPartition("bar", 0)); + checkRequest(createFetchRequest(7, new FetchMetadata(123, 456), toForgetTopics)); + checkResponse(createFetchResponse(123), 7); + checkResponse(createFetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, 123), 7); checkErrorResponse(createFetchRequest(4), new UnknownServerException()); checkRequest(createHeartBeatRequest()); checkErrorResponse(createHeartBeatRequest(), new UnknownServerException()); @@ -459,8 +467,8 @@ public void fetchResponseVersionTest() { responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE, 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); - FetchResponse v0Response = new FetchResponse(responseData, 0); - FetchResponse v1Response = new FetchResponse(responseData, 10); + FetchResponse v0Response = new FetchResponse(Errors.NONE, responseData, 0, INVALID_SESSION_ID); + FetchResponse v1Response = new FetchResponse(Errors.NONE, responseData, 10, INVALID_SESSION_ID); assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs()); assertEquals("Should use schema version 0", ApiKeys.FETCH.responseSchema((short) 0), @@ -488,15 +496,22 @@ public void testFetchResponseV4() { responseData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData(Errors.NONE, 70000, 6, FetchResponse.INVALID_LOG_START_OFFSET, Collections.emptyList(), records)); - FetchResponse response = new FetchResponse(responseData, 10); + FetchResponse response = new FetchResponse(Errors.NONE, responseData, 10, INVALID_SESSION_ID); FetchResponse deserialized = FetchResponse.parse(toBuffer(response.toStruct((short) 4)), (short) 4); assertEquals(responseData, deserialized.responseData()); } @Test - public void verifyFetchResponseFullWrite() throws Exception { - FetchResponse fetchResponse = createFetchResponse(); - short apiVersion = ApiKeys.FETCH.latestVersion(); + public void verifyFetchResponseFullWrites() throws Exception { + verifyFetchResponseFullWrite(ApiKeys.FETCH.latestVersion(), createFetchResponse(123)); + verifyFetchResponseFullWrite(ApiKeys.FETCH.latestVersion(), + createFetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, 123)); + for (short version = 0; version <= ApiKeys.FETCH.latestVersion(); version++) { + verifyFetchResponseFullWrite(version, createFetchResponse()); + } + } + + private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception { int correlationId = 15; Send send = fetchResponse.toSend("1", new ResponseHeader(correlationId), apiVersion); @@ -558,6 +573,19 @@ public void testFetchRequestIsolationLevel() throws Exception { assertEquals(request.isolationLevel(), deserialized.isolationLevel()); } + @Test + public void testFetchRequestWithMetadata() throws Exception { + FetchRequest request = createFetchRequest(4, IsolationLevel.READ_COMMITTED); + Struct struct = request.toStruct(); + FetchRequest deserialized = (FetchRequest) deserialize(request, struct, request.version()); + assertEquals(request.isolationLevel(), deserialized.isolationLevel()); + + request = createFetchRequest(4, IsolationLevel.READ_UNCOMMITTED); + struct = request.toStruct(); + deserialized = (FetchRequest) deserialize(request, struct, request.version()); + assertEquals(request.isolationLevel(), deserialized.isolationLevel()); + } + @Test public void testJoinGroupRequestVersion0RebalanceTimeout() throws Exception { final short version = 0; @@ -589,11 +617,20 @@ private FindCoordinatorResponse createFindCoordinatorResponse() { return new FindCoordinatorResponse(Errors.NONE, new Node(10, "host1", 2014)); } + private FetchRequest createFetchRequest(int version, FetchMetadata metadata, List toForget) { + LinkedHashMap fetchData = new LinkedHashMap<>(); + fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L, 1000000)); + fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L, 1000000)); + return FetchRequest.Builder.forConsumer(100, 100000, fetchData). + metadata(metadata).setMaxBytes(1000).toForget(toForget).build((short) version); + } + private FetchRequest createFetchRequest(int version, IsolationLevel isolationLevel) { LinkedHashMap fetchData = new LinkedHashMap<>(); fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L, 1000000)); fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L, 1000000)); - return FetchRequest.Builder.forConsumer(100, 100000, fetchData, isolationLevel).setMaxBytes(1000).build((short) version); + return FetchRequest.Builder.forConsumer(100, 100000, fetchData). + isolationLevel(isolationLevel).setMaxBytes(1000).build((short) version); } private FetchRequest createFetchRequest(int version) { @@ -603,6 +640,23 @@ private FetchRequest createFetchRequest(int version) { return FetchRequest.Builder.forConsumer(100, 100000, fetchData).setMaxBytes(1000).build((short) version); } + private FetchResponse createFetchResponse(Errors error, int sessionId) { + return new FetchResponse(error, new LinkedHashMap(), + 25, sessionId); + } + + private FetchResponse createFetchResponse(int sessionId) { + LinkedHashMap responseData = new LinkedHashMap<>(); + MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes())); + responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE, + 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); + List abortedTransactions = Collections.singletonList( + new FetchResponse.AbortedTransaction(234L, 999L)); + responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData(Errors.NONE, + 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, abortedTransactions, MemoryRecords.EMPTY)); + return new FetchResponse(Errors.NONE, responseData, 25, sessionId); + } + private FetchResponse createFetchResponse() { LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes())); @@ -614,7 +668,7 @@ private FetchResponse createFetchResponse() { responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData(Errors.NONE, 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, abortedTransactions, MemoryRecords.EMPTY)); - return new FetchResponse(responseData, 25); + return new FetchResponse(Errors.NONE, responseData, 25, INVALID_SESSION_ID); } private HeartbeatRequest createHeartBeatRequest() { diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java new file mode 100644 index 0000000000000..20084a266f208 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.utils; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; + +/** + * A unit test for ImplicitLinkedHashSet. + */ +public class ImplicitLinkedHashSetTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + private final static class TestElement implements ImplicitLinkedHashSet.Element { + private int prev = ImplicitLinkedHashSet.INVALID_INDEX; + private int next = ImplicitLinkedHashSet.INVALID_INDEX; + private final int val; + + TestElement(int val) { + this.val = val; + } + + @Override + public int prev() { + return prev; + } + + @Override + public void setPrev(int prev) { + this.prev = prev; + } + + @Override + public int next() { + return next; + } + + @Override + public void setNext(int next) { + this.next = next; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o == null) || (o.getClass() != TestElement.class)) return false; + TestElement that = (TestElement) o; + return val == that.val; + } + + @Override + public String toString() { + return "TestElement(" + val + ")"; + } + + @Override + public int hashCode() { + return val; + } + } + + @Test + public void testInsertDelete() throws Exception { + ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(100); + assertTrue(set.add(new TestElement(1))); + TestElement second = new TestElement(2); + assertTrue(set.add(second)); + assertTrue(set.add(new TestElement(3))); + assertFalse(set.add(new TestElement(3))); + assertEquals(3, set.size()); + assertTrue(set.contains(new TestElement(1))); + assertFalse(set.contains(new TestElement(4))); + TestElement secondAgain = set.find(new TestElement(2)); + assertTrue(second == secondAgain); + assertTrue(set.remove(new TestElement(1))); + assertFalse(set.remove(new TestElement(1))); + assertEquals(2, set.size()); + set.clear(); + assertEquals(0, set.size()); + } + + private static void expectTraversal(Iterator iterator, Integer... sequence) { + int i = 0; + while (iterator.hasNext()) { + TestElement element = iterator.next(); + Assert.assertTrue("Iterator yieled " + (i + 1) + " elements, but only " + + sequence.length + " were expected.", i < sequence.length); + Assert.assertEquals("Iterator value number " + (i + 1) + " was incorrect.", + sequence[i].intValue(), element.val); + i = i + 1; + } + Assert.assertTrue("Iterator yieled " + (i + 1) + " elements, but " + + sequence.length + " were expected.", i == sequence.length); + } + + private static void expectTraversal(Iterator iter, + Iterator expectedIter) { + int i = 0; + while (iter.hasNext()) { + TestElement element = iter.next(); + Assert.assertTrue("Iterator yieled " + (i + 1) + " elements, but only " + + i + " were expected.", expectedIter.hasNext()); + Integer expected = expectedIter.next(); + Assert.assertEquals("Iterator value number " + (i + 1) + " was incorrect.", + expected.intValue(), element.val); + i = i + 1; + } + Assert.assertFalse("Iterator yieled " + i + " elements, but at least " + + (i + 1) + " were expected.", expectedIter.hasNext()); + } + + @Test + public void testTraversal() throws Exception { + ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(100); + expectTraversal(set.iterator()); + assertTrue(set.add(new TestElement(2))); + expectTraversal(set.iterator(), 2); + assertTrue(set.add(new TestElement(1))); + expectTraversal(set.iterator(), 2, 1); + assertTrue(set.add(new TestElement(100))); + expectTraversal(set.iterator(), 2, 1, 100); + assertTrue(set.remove(new TestElement(1))); + expectTraversal(set.iterator(), 2, 100); + assertTrue(set.add(new TestElement(1))); + expectTraversal(set.iterator(), 2, 100, 1); + Iterator iter = set.iterator(); + iter.next(); + iter.next(); + iter.remove(); + iter.next(); + assertFalse(iter.hasNext()); + expectTraversal(set.iterator(), 2, 1); + List list = new ArrayList<>(); + list.add(new TestElement(1)); + list.add(new TestElement(2)); + assertTrue(set.removeAll(list)); + assertFalse(set.removeAll(list)); + expectTraversal(set.iterator()); + assertEquals(0, set.size()); + assertTrue(set.isEmpty()); + } + + @Test + public void testCollisions() throws Exception { + ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(5); + assertEquals(11, set.numSlots()); + assertTrue(set.add(new TestElement(11))); + assertTrue(set.add(new TestElement(0))); + assertTrue(set.add(new TestElement(22))); + assertTrue(set.add(new TestElement(33))); + assertEquals(11, set.numSlots()); + expectTraversal(set.iterator(), 11, 0, 22, 33); + assertTrue(set.remove(new TestElement(22))); + expectTraversal(set.iterator(), 11, 0, 33); + assertEquals(3, set.size()); + assertFalse(set.isEmpty()); + } + + @Test + public void testEnlargement() throws Exception { + ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(5); + assertEquals(11, set.numSlots()); + for (int i = 0; i < 6; i++) { + assertTrue(set.add(new TestElement(i))); + } + assertEquals(23, set.numSlots()); + assertEquals(6, set.size()); + expectTraversal(set.iterator(), 0, 1, 2, 3, 4, 5); + for (int i = 0; i < 6; i++) { + assertTrue("Failed to find element " + i, set.contains(new TestElement(i))); + } + set.remove(new TestElement(3)); + assertEquals(23, set.numSlots()); + assertEquals(5, set.size()); + expectTraversal(set.iterator(), 0, 1, 2, 4, 5); + } + + @Test + public void testManyInsertsAndDeletes() throws Exception { + Random random = new Random(123); + LinkedHashSet existing = new LinkedHashSet<>(); + ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(); + for (int i = 0; i < 100; i++) { + addRandomElement(random, existing, set); + addRandomElement(random, existing, set); + addRandomElement(random, existing, set); + removeRandomElement(random, existing, set); + expectTraversal(set.iterator(), existing.iterator()); + } + } + + private void addRandomElement(Random random, LinkedHashSet existing, + ImplicitLinkedHashSet set) { + int next; + do { + next = random.nextInt(); + } while (existing.contains(next)); + existing.add(next); + set.add(new TestElement(next)); + } + + private void removeRandomElement(Random random, LinkedHashSet existing, + ImplicitLinkedHashSet set) { + int removeIdx = random.nextInt(existing.size()); + Iterator iter = existing.iterator(); + Integer element = null; + for (int i = 0; i <= removeIdx; i++) { + element = iter.next(); + } + existing.remove(new TestElement(element)); + } +} diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala index f95fb89279937..b8329c1ece218 100644 --- a/core/src/main/scala/kafka/api/ApiVersion.scala +++ b/core/src/main/scala/kafka/api/ApiVersion.scala @@ -73,8 +73,10 @@ object ApiVersion { // Introduced LeaderAndIsrRequest V1, UpdateMetadataRequest V4 and FetchRequest V6 via KIP-112 "1.0-IV0" -> KAFKA_1_0_IV0, "1.0" -> KAFKA_1_0_IV0, - // Introduced DeleteGroupsRequest V0 via KIP-229 - "1.1-IV0" -> KAFKA_1_1_IV0 + // Introduced DeleteGroupsRequest V0 via KIP-229, plus KIP-227 incremental fetch requests, + // and KafkaStorageException for fetch requests. + "1.1-IV0" -> KAFKA_1_1_IV0, + "1.1" -> KAFKA_1_1_IV0 ) private val versionPattern = "\\.".r @@ -191,4 +193,3 @@ case object KAFKA_1_1_IV0 extends ApiVersion { val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V2 val id: Int = 14 } - diff --git a/core/src/main/scala/kafka/server/FetchSession.scala b/core/src/main/scala/kafka/server/FetchSession.scala new file mode 100644 index 0000000000000..0a825f1f18720 --- /dev/null +++ b/core/src/main/scala/kafka/server/FetchSession.scala @@ -0,0 +1,720 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import java.util +import java.util.concurrent.{ThreadLocalRandom, TimeUnit} + +import com.yammer.metrics.core.Gauge +import kafka.metrics.KafkaMetricsGroup +import kafka.utils.Logging +import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.requests.FetchMetadata.{FINAL_EPOCH, INITIAL_EPOCH, INVALID_SESSION_ID} +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} +import org.apache.kafka.common.requests.{FetchMetadata => JFetchMetadata} +import org.apache.kafka.common.utils.{ImplicitLinkedHashSet, Time, Utils} + +import scala.math.Ordered.orderingToOrdered +import scala.collection.{mutable, _} +import scala.collection.JavaConverters._ + +object FetchSession { + type REQ_MAP = util.Map[TopicPartition, FetchRequest.PartitionData] + type RESP_MAP = util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + type CACHE_MAP = ImplicitLinkedHashSet[CachedPartition] + + val NUM_INCREMENTAL_FETCH_SESSISONS = "NumIncrementalFetchSessions" + val NUM_INCREMENTAL_FETCH_PARTITIONS_CACHED = "NumIncrementalFetchPartitionsCached" + val INCREMENTAL_FETCH_SESSIONS_EVICTIONS_PER_SEC = "IncrementalFetchSessionEvictionsPerSec" + val EVICTIONS = "evictions" + + def partitionsToLogString(partitions: util.Collection[TopicPartition], traceEnabled: Boolean): String = { + if (traceEnabled) { + "(" + Utils.join(partitions, ", ") + ")" + } else { + s"${partitions.size} partition(s)" + } + } +} + +/** + * A cached partition. + * + * The broker maintains a set of these objects for each incremental fetch session. + * When an incremental fetch request is made, any partitions which are not explicitly + * enumerated in the fetch request are loaded from the cache. Similarly, when an + * incremental fetch response is being prepared, any partitions that have not changed + * are left out of the response. + * + * We store many of these objects, so it is important for them to be memory-efficient. + * That is why we store topic and partition separately rather than storing a TopicPartition + * object. The TP object takes up more memory because it is a separate JVM object, and + * because it stores the cached hash code in memory. + * + * Note that fetcherLogStartOffset is the LSO of the follower performing the fetch, whereas + * localLogStartOffset is the log start offset of the partition on this broker. + */ +class CachedPartition(val topic: String, + val partition: Int, + var maxBytes: Int, + var fetchOffset: Long, + var highWatermark: Long, + var fetcherLogStartOffset: Long, + var localLogStartOffset: Long) + extends ImplicitLinkedHashSet.Element { + + var cachedNext: Int = ImplicitLinkedHashSet.INVALID_INDEX + var cachedPrev: Int = ImplicitLinkedHashSet.INVALID_INDEX + + override def next = cachedNext + override def setNext(next: Int) = this.cachedNext = next + override def prev = cachedPrev + override def setPrev(prev: Int) = this.cachedPrev = prev + + def this(topic: String, partition: Int) = + this(topic, partition, -1, -1, -1, -1, -1) + + def this(part: TopicPartition) = + this(part.topic(), part.partition()) + + def this(part: TopicPartition, reqData: FetchRequest.PartitionData) = + this(part.topic(), part.partition(), + reqData.maxBytes, reqData.fetchOffset, -1, + reqData.logStartOffset, -1) + + def this(part: TopicPartition, reqData: FetchRequest.PartitionData, + respData: FetchResponse.PartitionData) = + this(part.topic(), part.partition(), + reqData.maxBytes, reqData.fetchOffset, respData.highWatermark, + reqData.logStartOffset, respData.logStartOffset) + + def topicPartition() = new TopicPartition(topic, partition) + + def reqData() = new FetchRequest.PartitionData(fetchOffset, fetcherLogStartOffset, maxBytes) + + def updateRequestParams(reqData: FetchRequest.PartitionData): Unit = { + // Update our cached request parameters. + maxBytes = reqData.maxBytes + fetchOffset = reqData.fetchOffset + fetcherLogStartOffset = reqData.logStartOffset + } + + /** + * Update this CachedPartition with new request and response data. + * + * This function should be called while holding the appropriate session + * lock. + * + * @return True if this partition should be included in the FetchResponse + * we send back to the fetcher; false if it can be omitted. + */ + def updateResponseData(respData: FetchResponse.PartitionData): Boolean = { + // Check the response data. + var mustRespond = false + if ((respData.records != null) && (respData.records.sizeInBytes() > 0)) { + // Partitions with new data are always included in the response. + mustRespond = true + } + if (highWatermark != respData.highWatermark) { + mustRespond = true + highWatermark = respData.highWatermark + } + if (localLogStartOffset != respData.logStartOffset) { + mustRespond = true + localLogStartOffset = respData.logStartOffset + } + if (respData.error.code() != 0) { + // Partitions with errors are always included in the response. + // We also set the cached highWatermark to an invalid offset, -1. + // This ensures that when the error goes away, we re-send the partition. + highWatermark = -1 + mustRespond = true + } + mustRespond + } + + override def hashCode() = (31 * partition) + topic.hashCode + + def canEqual(that: Any) = that.isInstanceOf[CachedPartition] + + override def equals(that: Any): Boolean = + that match { + case that: CachedPartition => that.canEqual(this) && + this.topic.equals(that.topic) && + this.partition.equals(that.partition) + case _ => false + } + + override def toString() = synchronized { + "CachedPartition(topic=" + topic + + ", partition=" + partition + + ", maxBytes=" + maxBytes + + ", fetchOffset=" + fetchOffset + + ", highWatermark=" + highWatermark + + ", fetcherLogStartOffset=" + fetcherLogStartOffset + + ", localLogStartOffset=" + localLogStartOffset + + ")" + } +} + +/** + * The fetch session. + * + * Each fetch session is protected by its own lock, which must be taken before mutable + * fields are read or modified. This includes modification of the session partition map. + * + * @param id The unique fetch session ID. + * @param privileged True if this session is privileged. Sessions crated by followers + * are privileged; sesssion created by consumers are not. + * @param partitionMap The CachedPartitionMap. + * @param creationMs The time in milliseconds when this session was created. + * @param lastUsedMs The last used time in milliseconds. This should only be updated by + * FetchSessionCache#touch. + * @param epoch The fetch session sequence number. + */ +case class FetchSession(val id: Int, + val privileged: Boolean, + val partitionMap: FetchSession.CACHE_MAP, + val creationMs: Long, + var lastUsedMs: Long, + var epoch: Int) { + // This is used by the FetchSessionCache to store the last known size of this session. + // If this is -1, the Session is not in the cache. + var cachedSize = -1 + + def size(): Int = synchronized { + partitionMap.size() + } + + def isEmpty(): Boolean = synchronized { + partitionMap.isEmpty + } + + def lastUsedKey(): LastUsedKey = synchronized { + LastUsedKey(lastUsedMs, id) + } + + def evictableKey(): EvictableKey = synchronized { + EvictableKey(privileged, cachedSize, id) + } + + def metadata(): JFetchMetadata = synchronized { new JFetchMetadata(id, epoch) } + + def getFetchOffset(topicPartition: TopicPartition): Option[Long] = synchronized { + Option(partitionMap.find(new CachedPartition(topicPartition))).map(_.fetchOffset) + } + + type TL = util.ArrayList[TopicPartition] + + // Update the cached partition data based on the request. + def update(fetchData: FetchSession.REQ_MAP, + toForget: util.List[TopicPartition], + reqMetadata: JFetchMetadata): (TL, TL, TL) = synchronized { + val added = new TL + val updated = new TL + val removed = new TL + fetchData.entrySet().iterator().asScala.foreach(entry => { + val topicPart = entry.getKey + val reqData = entry.getValue + val newCachedPart = new CachedPartition(topicPart, reqData) + val cachedPart = partitionMap.find(newCachedPart) + if (cachedPart == null) { + partitionMap.mustAdd(newCachedPart) + added.add(topicPart) + } else { + cachedPart.updateRequestParams(reqData) + updated.add(topicPart) + } + }) + toForget.iterator().asScala.foreach(p => { + if (partitionMap.remove(new CachedPartition(p.topic(), p.partition()))) { + removed.add(p) + } + }) + (added, updated, removed) + } + + override def toString(): String = synchronized { + "FetchSession(id=" + id + + ", privileged=" + privileged + + ", partitionMap.size=" + partitionMap.size() + + ", creationMs=" + creationMs + + ", creationMs=" + lastUsedMs + + ", epoch=" + epoch + ")" + } +} + +trait FetchContext extends Logging { + /** + * Get the fetch offset for a given partition. + */ + def getFetchOffset(part: TopicPartition): Option[Long] + + /** + * Apply a function to each partition in the fetch request. + */ + def foreachPartition(fun: (TopicPartition, FetchRequest.PartitionData) => Unit): Unit + + /** + * Updates the fetch context with new partition information. Generates response data. + * The response data may require subsequent down-conversion. + */ + def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse + + def partitionsToLogString(partitions: util.Collection[TopicPartition]): String = + FetchSession.partitionsToLogString(partitions, isTraceEnabled) +} + +/** + * The fetch context for a fetch request that had a session error. + */ +class SessionErrorContext(val error: Errors, + val reqMetadata: JFetchMetadata) extends FetchContext { + override def getFetchOffset(part: TopicPartition): Option[Long] = None + + override def foreachPartition(fun: (TopicPartition, FetchRequest.PartitionData) => Unit): Unit = {} + + // Because of the fetch session error, we don't know what partitions were supposed to be in this request. + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { + debug(s"Session error fetch context returning $error") + new FetchResponse(error, new FetchSession.RESP_MAP, 0, INVALID_SESSION_ID) + } +} + +/** + * The fetch context for a sessionless fetch request. + * + * @param fetchData The partition data from the fetch request. + */ +class SessionlessFetchContext(val fetchData: util.Map[TopicPartition, FetchRequest.PartitionData]) extends FetchContext { + override def getFetchOffset(part: TopicPartition): Option[Long] = + Option(fetchData.get(part)).map(_.fetchOffset) + + override def foreachPartition(fun: (TopicPartition, FetchRequest.PartitionData) => Unit): Unit = { + fetchData.entrySet().asScala.foreach(entry => fun(entry.getKey, entry.getValue)) + } + + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { + debug(s"Sessionless fetch context returning ${partitionsToLogString(updates.keySet())}") + new FetchResponse(Errors.NONE, updates, 0, INVALID_SESSION_ID) + } +} + +/** + * The fetch context for a full fetch request. + * + * @param time The clock to use. + * @param cache The fetch session cache. + * @param reqMetadata The request metadata. + * @param fetchData The partition data from the fetch request. + * @param isFromFollower True if this fetch request came from a follower. + */ +class FullFetchContext(private val time: Time, + private val cache: FetchSessionCache, + private val reqMetadata: JFetchMetadata, + private val fetchData: util.Map[TopicPartition, FetchRequest.PartitionData], + private val isFromFollower: Boolean) extends FetchContext { + override def getFetchOffset(part: TopicPartition): Option[Long] = + Option(fetchData.get(part)).map(_.fetchOffset) + + override def foreachPartition(fun: (TopicPartition, FetchRequest.PartitionData) => Unit): Unit = { + fetchData.entrySet().asScala.foreach(entry => fun(entry.getKey, entry.getValue)) + } + + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { + def createNewSession(): FetchSession.CACHE_MAP = { + val cachedPartitions = new FetchSession.CACHE_MAP(updates.size()) + updates.entrySet().asScala.foreach(entry => { + val part = entry.getKey + val respData = entry.getValue + val reqData = fetchData.get(part) + cachedPartitions.mustAdd(new CachedPartition(part, reqData, respData)) + }) + cachedPartitions + } + val responseSessionId = cache.maybeCreateSession(time.milliseconds(), isFromFollower, + updates.size(), createNewSession) + debug(s"Full fetch context with session id $responseSessionId returning " + + s"${partitionsToLogString(updates.keySet())}") + new FetchResponse(Errors.NONE, updates, 0, responseSessionId) + } +} + +/** + * The fetch context for an incremental fetch request. + * + * @param time The clock to use. + * @param reqMetadata The request metadata. + * @param session The incremental fetch request session. + */ +class IncrementalFetchContext(private val time: Time, + private val reqMetadata: JFetchMetadata, + private val session: FetchSession) extends FetchContext { + + override def getFetchOffset(tp: TopicPartition): Option[Long] = session.getFetchOffset(tp) + + override def foreachPartition(fun: (TopicPartition, FetchRequest.PartitionData) => Unit): Unit = { + // Take the session lock and iterate over all the cached partitions. + session.synchronized { + session.partitionMap.iterator().asScala.foreach(part => { + fun(new TopicPartition(part.topic, part.partition), part.reqData()) + }) + } + } + + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { + session.synchronized { + // Check to make sure that the session epoch didn't change in between + // creating this fetch context and generating this response. + val expectedEpoch = JFetchMetadata.nextEpoch(reqMetadata.epoch()) + if (session.epoch != expectedEpoch) { + info(s"Incremental fetch session ${session.id} expected epoch $expectedEpoch, but " + + s"got ${session.epoch}. Possible duplicate request.") + new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, new FetchSession.RESP_MAP, 0, session.id) + } else { + // Iterate over the update list. Prune updates which don't need to be sent. + val iter = updates.entrySet().iterator() + while (iter.hasNext()) { + val entry = iter.next() + val topicPart = entry.getKey + val respData = entry.getValue + val cachedPart = session.partitionMap.find(new CachedPartition(topicPart)) + val mustRespond = cachedPart.updateResponseData(respData) + if (mustRespond) { + // Move this to the end of the cached partition map. + // This is important for ensuring fairness when lots of partitions + // have data to return. + session.partitionMap.remove(cachedPart) + session.partitionMap.mustAdd(cachedPart) + } else { + // Do not include this partition in the FetchResponse. + iter.remove() + } + } + debug(s"Incremental fetch context with session id ${session.id} returning " + + s"${partitionsToLogString(updates.keySet())}") + new FetchResponse(Errors.NONE, updates, 0, session.id) + } + } + } +} + +case class LastUsedKey(val lastUsedMs: Long, + val id: Int) extends Comparable[LastUsedKey] { + override def compareTo(other: LastUsedKey): Int = + (lastUsedMs, id) compare (other.lastUsedMs, other.id) +} + +case class EvictableKey(val privileged: Boolean, + val size: Int, + val id: Int) extends Comparable[EvictableKey] { + override def compareTo(other: EvictableKey): Int = + (privileged, size, id) compare (other.privileged, other.size, other.id) +} + +/** + * Caches fetch sessions. + * + * See tryEvict for an explanation of the cache eviction strategy. + * + * The FetchSessionCache is thread-safe because all of its methods are synchronized. + * Note that individual fetch sessions have their own locks which are separate from the + * FetchSessionCache lock. In order to avoid deadlock, the FetchSessionCache lock + * must never be acquired while an individual FetchSession lock is already held. + * + * @param maxEntries The maximum number of entries that can be in the cache. + * @param evictionMs The minimum time that an entry must be unused in order to be evictable. + */ +class FetchSessionCache(private val maxEntries: Int, + private val evictionMs: Long) extends Logging with KafkaMetricsGroup { + private var numPartitions: Long = 0 + + // A map of session ID to FetchSession. + private val sessions = new mutable.HashMap[Int, FetchSession] + + // Maps last used times to sessions. + private val lastUsed = new util.TreeMap[LastUsedKey, FetchSession] + + // A map containing sessions which can be evicted by both privileged and + // unprivileged sessions. + private val evictableByAll = new util.TreeMap[EvictableKey, FetchSession] + + // A map containing sessions which can be evicted by privileged sessions. + private val evictableByPrivileged = new util.TreeMap[EvictableKey, FetchSession] + + // Set up metrics. + removeMetric(FetchSession.NUM_INCREMENTAL_FETCH_SESSISONS) + newGauge(FetchSession.NUM_INCREMENTAL_FETCH_SESSISONS, + new Gauge[Int] { + def value = FetchSessionCache.this.size + } + ) + removeMetric(FetchSession.NUM_INCREMENTAL_FETCH_PARTITIONS_CACHED) + newGauge(FetchSession.NUM_INCREMENTAL_FETCH_PARTITIONS_CACHED, + new Gauge[Long] { + def value = FetchSessionCache.this.totalPartitions + } + ) + removeMetric(FetchSession.INCREMENTAL_FETCH_SESSIONS_EVICTIONS_PER_SEC) + val evictionsMeter = newMeter(FetchSession.INCREMENTAL_FETCH_SESSIONS_EVICTIONS_PER_SEC, + FetchSession.EVICTIONS, TimeUnit.SECONDS, Map.empty) + + /** + * Get a session by session ID. + * + * @param sessionId The session ID. + * @return The session, or None if no such session was found. + */ + def get(sessionId: Int): Option[FetchSession] = synchronized { + sessions.get(sessionId) + } + + /** + * Get the number of entries currently in the fetch session cache. + */ + def size(): Int = synchronized { + sessions.size + } + + /** + * Get the total number of cached partitions. + */ + def totalPartitions(): Long = synchronized { + numPartitions + } + + /** + * Creates a new random session ID. The new session ID will be positive and unique on this broker. + * + * @return The new session ID. + */ + def newSessionId(): Int = synchronized { + var id = 0 + do { + id = ThreadLocalRandom.current().nextInt(1, Int.MaxValue) + } while (sessions.contains(id) || id == INVALID_SESSION_ID) + id + } + + /** + * Try to create a new session. + * + * @param now The current time in milliseconds. + * @param privileged True if the new entry we are trying to create is privileged. + * @param size The number of cached partitions in the new entry we are trying to create. + * @param createPartitions A callback function which creates the map of cached partitions. + * @return If we created a session, the ID; INVALID_SESSION_ID otherwise. + */ + def maybeCreateSession(now: Long, + privileged: Boolean, + size: Int, + createPartitions: () => FetchSession.CACHE_MAP): Int = + synchronized { + // If there is room, create a new session entry. + if ((sessions.size < maxEntries) || + tryEvict(privileged, EvictableKey(privileged, size, 0), now)) { + val partitionMap = createPartitions() + val session = new FetchSession(newSessionId(), privileged, partitionMap, + now, now, JFetchMetadata.nextEpoch(INITIAL_EPOCH)) + debug(s"Created fetch session ${session.toString()}") + sessions.put(session.id, session) + touch(session, now) + session.id + } else { + debug(s"No fetch session created for privileged=$privileged, size=$size.") + INVALID_SESSION_ID + } + } + + /** + * Try to evict an entry from the session cache. + * + * A proposed new element A may evict an existing element B if: + * 1. A is privileged and B is not, or + * 2. B is considered "stale" because it has been inactive for a long time, or + * 3. A contains more partitions than B, and B is not recently created. + * + * @param privileged True if the new entry we would like to add is privileged. + * @param key The EvictableKey for the new entry we would like to add. + * @param now The current time in milliseconds. + * @return True if an entry was evicted; false otherwise. + */ + def tryEvict(privileged: Boolean, key: EvictableKey, now: Long): Boolean = synchronized { + // Try to evict an entry which is stale. + val lastUsedEntry = lastUsed.firstEntry() + if (lastUsedEntry == null) { + trace("There are no cache entries to evict.") + false + } else if (now - lastUsedEntry.getKey().lastUsedMs > evictionMs) { + val session = lastUsedEntry.getValue() + trace(s"Evicting stale FetchSession ${session.id}.") + remove(session) + evictionsMeter.mark() + true + } else { + // If there are no stale entries, check the first evictable entry. + // If it is less valuable than our proposed entry, evict it. + val map = if (privileged) evictableByPrivileged else evictableByAll + val evictableEntry = map.firstEntry() + if (evictableEntry == null) { + trace("No evictable entries found.") + false + } else if (key.compareTo(evictableEntry.getKey()) < 0) { + trace(s"Can't evict ${evictableEntry.getKey()} with ${key.toString}") + false + } else { + trace(s"Evicting ${evictableEntry.getKey()} with ${key.toString}.") + remove(evictableEntry.getValue()) + evictionsMeter.mark() + true + } + } + } + + def remove(sessionId: Int): Option[FetchSession] = synchronized { + get(sessionId) match { + case None => None + case Some(session) => remove(session) + } + } + + /** + * Remove an entry from the session cache. + * + * @param session The session. + * + * @return The removed session, or None if there was no such session. + */ + def remove(session: FetchSession): Option[FetchSession] = synchronized { + val evictableKey = session.synchronized { + lastUsed.remove(session.lastUsedKey()) + session.evictableKey() + } + evictableByAll.remove(evictableKey) + evictableByPrivileged.remove(evictableKey) + val removeResult = sessions.remove(session.id) + if (removeResult.isDefined) { + numPartitions = numPartitions - session.cachedSize + } + removeResult + } + + /** + * Update a session's position in the lastUsed and evictable trees. + * + * @param session The session. + * @param now The current time in milliseconds. + */ + def touch(session: FetchSession, now: Long): Unit = synchronized { + session.synchronized { + // Update the lastUsed map. + lastUsed.remove(session.lastUsedKey()) + session.lastUsedMs = now + lastUsed.put(session.lastUsedKey(), session) + + val oldSize = session.cachedSize + if (oldSize != -1) { + val oldEvictableKey = session.evictableKey() + evictableByPrivileged.remove(oldEvictableKey) + evictableByAll.remove(oldEvictableKey) + numPartitions = numPartitions - oldSize + } + session.cachedSize = session.size() + val newEvictableKey = session.evictableKey() + if ((!session.privileged) || (now - session.creationMs > evictionMs)) { + evictableByPrivileged.put(newEvictableKey, session) + } + if (now - session.creationMs > evictionMs) { + evictableByAll.put(newEvictableKey, session) + } + numPartitions = numPartitions + session.cachedSize + } + } +} + +class FetchManager(private val time: Time, + private val cache: FetchSessionCache) extends Logging { + def newContext(reqMetadata: JFetchMetadata, + fetchData: FetchSession.REQ_MAP, + toForget: util.List[TopicPartition], + isFollower: Boolean): FetchContext = { + val context = if (reqMetadata.isFull) { + var removedFetchSessionStr = "" + if (reqMetadata.sessionId() != INVALID_SESSION_ID) { + // Any session specified in a FULL fetch request will be closed. + if (cache.remove(reqMetadata.sessionId()).isDefined) { + removedFetchSessionStr = s" Removed fetch session ${reqMetadata.sessionId()}." + } + } + var suffix = "" + val context = if (reqMetadata.epoch() == FINAL_EPOCH) { + // If the epoch is FINAL_EPOCH, don't try to create a new session. + suffix = " Will not try to create a new session." + new SessionlessFetchContext(fetchData) + } else { + new FullFetchContext(time, cache, reqMetadata, fetchData, isFollower) + } + debug(s"Created a new full FetchContext with ${partitionsToLogString(fetchData.keySet())}."+ + s"${removedFetchSessionStr}${suffix}") + context + } else { + cache.synchronized { + cache.get(reqMetadata.sessionId()) match { + case None => { + info(s"Created a new error FetchContext for session id ${reqMetadata.sessionId()}: " + + "no such session ID found.") + new SessionErrorContext(Errors.FETCH_SESSION_ID_NOT_FOUND, reqMetadata) + } + case Some(session) => session.synchronized { + if (session.epoch != reqMetadata.epoch()) { + debug(s"Created a new error FetchContext for session id ${session.id}: expected " + + s"epoch ${session.epoch}, but got epoch ${reqMetadata.epoch()}.") + new SessionErrorContext(Errors.INVALID_FETCH_SESSION_EPOCH, reqMetadata) + } else { + val (added, updated, removed) = session.update(fetchData, toForget, reqMetadata) + if (session.isEmpty) { + debug(s"Created a new sessionless FetchContext and closing session id ${session.id}, " + + s"epoch ${session.epoch}: after removing ${partitionsToLogString(removed)}, " + + s"there are no more partitions left.") + cache.remove(session) + new SessionlessFetchContext(fetchData) + } else { + if (session.size() != session.cachedSize) { + // If the number of partitions in the session changed, update the session's + // position in the cache. + cache.touch(session, session.lastUsedMs) + } + session.epoch = JFetchMetadata.nextEpoch(session.epoch) + debug(s"Created a new incremental FetchContext for session id ${session.id}, " + + s"epoch ${session.epoch}: added ${partitionsToLogString(added)}, " + + s"updated ${partitionsToLogString(updated)}, " + + s"removed ${partitionsToLogString(removed)}") + new IncrementalFetchContext(time, reqMetadata, session) + } + } + } + } + } + } + context + } + + def partitionsToLogString(partitions: util.Collection[TopicPartition]): String = + FetchSession.partitionsToLogString(partitions, isTraceEnabled) +} diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 1f448af71a099..b84587f55c293 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -80,6 +80,7 @@ class KafkaApis(val requestChannel: RequestChannel, val metrics: Metrics, val authorizer: Option[Authorizer], val quotas: QuotaManagers, + val fetchManager: FetchManager, brokerTopicStats: BrokerTopicStats, val clusterId: String, time: Time, @@ -481,35 +482,52 @@ class KafkaApis(val requestChannel: RequestChannel, * Handle a fetch request */ def handleFetchRequest(request: RequestChannel.Request) { - val fetchRequest = request.body[FetchRequest] val versionId = request.header.apiVersion val clientId = request.header.clientId - - val unauthorizedTopicResponseData = mutable.ArrayBuffer[(TopicPartition, FetchResponse.PartitionData)]() - val nonExistingTopicResponseData = mutable.ArrayBuffer[(TopicPartition, FetchResponse.PartitionData)]() - val authorizedRequestInfo = mutable.ArrayBuffer[(TopicPartition, FetchRequest.PartitionData)]() - - if (fetchRequest.isFromFollower() && !authorize(request.session, ClusterAction, Resource.ClusterResource)) - for (topicPartition <- fetchRequest.fetchData.asScala.keys) - unauthorizedTopicResponseData += topicPartition -> new FetchResponse.PartitionData(Errors.CLUSTER_AUTHORIZATION_FAILED, - FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) - else - for ((topicPartition, partitionData) <- fetchRequest.fetchData.asScala) { - if (!authorize(request.session, Read, new Resource(Topic, topicPartition.topic))) - unauthorizedTopicResponseData += topicPartition -> new FetchResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED, + val fetchRequest = request.body[FetchRequest] + val fetchContext = fetchManager.newContext(fetchRequest.metadata(), + fetchRequest.fetchData(), + fetchRequest.toForget(), + fetchRequest.isFromFollower()) + + val erroneous = mutable.ArrayBuffer[(TopicPartition, FetchResponse.PartitionData)]() + val interesting = mutable.ArrayBuffer[(TopicPartition, FetchRequest.PartitionData)]() + if (fetchRequest.isFromFollower()) { + // The follower must have ClusterAction on ClusterResource in order to fetch partition data. + if (authorize(request.session, ClusterAction, Resource.ClusterResource)) { + fetchContext.foreachPartition((part, data) => { + if (!metadataCache.contains(part.topic)) { + erroneous += part -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION, + FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, + FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) + } else { + interesting += (part -> data) + } + }) + } else { + fetchContext.foreachPartition((part, data) => { + erroneous += part -> new FetchResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED, FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) - else if (!metadataCache.contains(topicPartition.topic)) - nonExistingTopicResponseData += topicPartition -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION, + }) + } + } else { + // Regular Kafka consumers need READ permission on each partition they are fetching. + fetchContext.foreachPartition((part, data) => { + if (!authorize(request.session, Read, new Resource(Topic, part.topic))) + erroneous += part -> new FetchResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED, + FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, + FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) + else if (!metadataCache.contains(part.topic)) + erroneous += part -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION, FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) else - authorizedRequestInfo += (topicPartition -> partitionData) - } + interesting += (part -> data) + }) + } def convertedPartitionData(tp: TopicPartition, data: FetchResponse.PartitionData) = { - // Down-conversion of the fetched records is needed when the stored magic version is // greater than that supported by the client (as indicated by the fetch request version). If the // configured magic version for the topic is less than or equal to that supported by the version of the @@ -529,7 +547,7 @@ class KafkaApis(val requestChannel: RequestChannel, downConvertMagic.map { magic => trace(s"Down converting records from partition $tp to message format version $magic for fetch request from $clientId") - val converted = data.records.downConvert(magic, fetchRequest.fetchData.get(tp).fetchOffset, time) + val converted = data.records.downConvert(magic, fetchContext.getFetchOffset(tp).get, time) updateRecordsProcessingStats(request, tp, converted.recordsProcessingStats) new FetchResponse.PartitionData(data.error, data.highWatermark, FetchResponse.INVALID_LAST_STABLE_OFFSET, data.logStartOffset, data.abortedTransactions, converted.records) @@ -540,34 +558,28 @@ class KafkaApis(val requestChannel: RequestChannel, // the callback for process a fetch response, invoked before throttling def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]) { - val partitionData = { - responsePartitionData.map { case (tp, data) => - val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull - val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) - tp -> new FetchResponse.PartitionData(data.error, data.highWatermark, lastStableOffset, - data.logStartOffset, abortedTransactions, data.records) - } - } - - val mergedPartitionData = partitionData ++ unauthorizedTopicResponseData ++ nonExistingTopicResponseData - val fetchedPartitionData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData]() - - mergedPartitionData.foreach { case (topicPartition, data) => - if (data.error != Errors.NONE) - debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " + - s"on partition $topicPartition failed due to ${data.error.exceptionName}") - - fetchedPartitionData.put(topicPartition, data) + val partitions = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + responsePartitionData.foreach{ case (tp, data) => + val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull + val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) + partitions.put(tp, new FetchResponse.PartitionData(data.error, data.highWatermark, lastStableOffset, + data.logStartOffset, abortedTransactions, data.records)) } + erroneous.foreach{case (tp, data) => partitions.put(tp, data)} + val unconvertedFetchResponse = fetchContext.updateAndGenerateResponseData(partitions) // fetch response callback invoked after any throttling def fetchResponseCallback(bandwidthThrottleTimeMs: Int) { def createResponse(requestThrottleTimeMs: Int): FetchResponse = { val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] - fetchedPartitionData.asScala.foreach { case (tp, partitionData) => + unconvertedFetchResponse.responseData().asScala.foreach { case (tp, partitionData) => + if (partitionData.error != Errors.NONE) + debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " + + s"on partition $tp failed due to ${partitionData.error.exceptionName}") convertedData.put(tp, convertedPartitionData(tp, partitionData)) } - val response = new FetchResponse(convertedData, bandwidthThrottleTimeMs + requestThrottleTimeMs) + val response = new FetchResponse(unconvertedFetchResponse.error(), convertedData, + bandwidthThrottleTimeMs + requestThrottleTimeMs, unconvertedFetchResponse.sessionId()) response.responseData.asScala.foreach { case (topicPartition, data) => // record the bytes out metrics only when the response is being sent brokerTopicStats.updateBytesOut(topicPartition.topic, fetchRequest.isFromFollower, data.records.sizeInBytes) @@ -575,6 +587,9 @@ class KafkaApis(val requestChannel: RequestChannel, response } + trace(s"Sending Fetch response with partitions.size=${unconvertedFetchResponse.responseData().size()}, " + + s"metadata=${unconvertedFetchResponse.sessionId()}") + if (fetchRequest.isFromFollower) sendResponseExemptThrottle(request, createResponse(0)) else @@ -587,21 +602,20 @@ class KafkaApis(val requestChannel: RequestChannel, if (fetchRequest.isFromFollower) { // We've already evaluated against the quota and are good to go. Just need to record it now. - val responseSize = sizeOfThrottledPartitions(versionId, fetchRequest, mergedPartitionData, quotas.leader) + val responseSize = sizeOfThrottledPartitions(versionId, unconvertedFetchResponse, quotas.leader) quotas.leader.record(responseSize) fetchResponseCallback(bandwidthThrottleTimeMs = 0) } else { // Fetch size used to determine throttle time is calculated before any down conversions. // This may be slightly different from the actual response size. But since down conversions // result in data being loaded into memory, it is better to do this after throttling to avoid OOM. - val response = new FetchResponse(fetchedPartitionData, 0) - val responseStruct = response.toStruct(versionId) + val responseStruct = unconvertedFetchResponse.toStruct(versionId) quotas.fetch.maybeRecordAndThrottle(request.session.sanitizedUser, clientId, responseStruct.sizeOf, fetchResponseCallback) } } - if (authorizedRequestInfo.isEmpty) + if (interesting.isEmpty) processResponseCallback(Seq.empty) else { // call the replica manager to fetch messages from the local replica @@ -611,23 +625,45 @@ class KafkaApis(val requestChannel: RequestChannel, fetchRequest.minBytes, fetchRequest.maxBytes, versionId <= 2, - authorizedRequestInfo, + interesting, replicationQuota(fetchRequest), processResponseCallback, fetchRequest.isolationLevel) } } + class SelectingIterator(val partitions: util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData], + val quota: ReplicationQuotaManager) + extends util.Iterator[util.Map.Entry[TopicPartition, FetchResponse.PartitionData]] { + val iter = partitions.entrySet().iterator() + + var nextElement: util.Map.Entry[TopicPartition, FetchResponse.PartitionData] = null + + override def hasNext: Boolean = { + while ((nextElement == null) && iter.hasNext()) { + val element = iter.next() + if (quota.isThrottled(element.getKey)) { + nextElement = element + } + } + nextElement != null + } + + override def next(): util.Map.Entry[TopicPartition, FetchResponse.PartitionData] = { + if (!hasNext()) throw new NoSuchElementException() + val element = nextElement + nextElement = null + element + } + + override def remove() = throw new UnsupportedOperationException() + } + private def sizeOfThrottledPartitions(versionId: Short, - fetchRequest: FetchRequest, - mergedPartitionData: Seq[(TopicPartition, FetchResponse.PartitionData)], + unconvertedResponse: FetchResponse, quota: ReplicationQuotaManager): Int = { - val partitionData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] - mergedPartitionData.foreach { case (tp, data) => - if (quota.isThrottled(tp)) - partitionData.put(tp, data) - } - FetchResponse.sizeOf(versionId, partitionData) + val iter = new SelectingIterator(unconvertedResponse.responseData(), quota) + FetchResponse.sizeOf(versionId, iter) } def replicationQuota(fetchRequest: FetchRequest): ReplicaQuota = diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 64698f7f3f690..0b9bdaa8bea10 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -174,6 +174,9 @@ object Defaults { val TransactionsAbortTimedOutTransactionsCleanupIntervalMS = TransactionStateManager.DefaultAbortTimedOutTransactionsIntervalMs val TransactionsRemoveExpiredTransactionsCleanupIntervalMS = TransactionStateManager.DefaultRemoveExpiredTransactionalIdsIntervalMs + /** ********* Fetch Session Configuration **************/ + val MaxIncrementalFetchSessionCacheSlots = 1000 + /** ********* Quota Configuration ***********/ val ProducerQuotaBytesPerSecondDefault = ClientQuotaManagerConfig.QuotaBytesPerSecondDefault val ConsumerQuotaBytesPerSecondDefault = ClientQuotaManagerConfig.QuotaBytesPerSecondDefault @@ -375,6 +378,9 @@ object KafkaConfig { val TransactionsAbortTimedOutTransactionCleanupIntervalMsProp = "transaction.abort.timed.out.transaction.cleanup.interval.ms" val TransactionsRemoveExpiredTransactionalIdCleanupIntervalMsProp = "transaction.remove.expired.transaction.cleanup.interval.ms" + /** ********* Fetch Session Configuration **************/ + val MaxIncrementalFetchSessionCacheSlots = "max.incremental.fetch.session.cache.slots" + /** ********* Quota Configuration ***********/ val ProducerQuotaBytesPerSecondDefaultProp = "quota.producer.default" val ConsumerQuotaBytesPerSecondDefaultProp = "quota.consumer.default" @@ -652,6 +658,9 @@ object KafkaConfig { val TransactionsAbortTimedOutTransactionsIntervalMsDoc = "The interval at which to rollback transactions that have timed out" val TransactionsRemoveExpiredTransactionsIntervalMsDoc = "The interval at which to remove transactions that have expired due to transactional.id.expiration.ms passing" + /** ********* Fetch Session Configuration **************/ + val MaxIncrementalFetchSessionCacheSlotsDoc = "The maximum number of incremental fetch sessions that we will maintain." + /** ********* Quota Configuration ***********/ val ProducerQuotaBytesPerSecondDefaultDoc = "DEPRECATED: Used only when dynamic default quotas are not configured for , or in Zookeeper. " + "Any producer distinguished by clientId will get throttled if it produces more bytes than this value per-second" @@ -886,6 +895,9 @@ object KafkaConfig { .define(TransactionsAbortTimedOutTransactionCleanupIntervalMsProp, INT, Defaults.TransactionsAbortTimedOutTransactionsCleanupIntervalMS, atLeast(1), LOW, TransactionsAbortTimedOutTransactionsIntervalMsDoc) .define(TransactionsRemoveExpiredTransactionalIdCleanupIntervalMsProp, INT, Defaults.TransactionsRemoveExpiredTransactionsCleanupIntervalMS, atLeast(1), LOW, TransactionsRemoveExpiredTransactionsIntervalMsDoc) + /** ********* Fetch Session Configuration **************/ + .define(MaxIncrementalFetchSessionCacheSlots, INT, Defaults.MaxIncrementalFetchSessionCacheSlots, atLeast(0), MEDIUM, MaxIncrementalFetchSessionCacheSlotsDoc) + /** ********* Kafka Metrics Configuration ***********/ .define(MetricNumSamplesProp, INT, Defaults.MetricNumSamples, atLeast(1), LOW, MetricNumSamplesDoc) .define(MetricSampleWindowMsProp, LONG, Defaults.MetricSampleWindowMs, atLeast(1), LOW, MetricSampleWindowMsDoc) @@ -1196,6 +1208,9 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO /** ********* Transaction Configuration **************/ val transactionIdExpirationMs = getInt(KafkaConfig.TransactionalIdExpirationMsProp) + /** ********* Fetch Session Configuration **************/ + val maxIncrementalFetchSessionCacheSlots = getInt(KafkaConfig.MaxIncrementalFetchSessionCacheSlots) + val deleteTopicEnable = getBoolean(KafkaConfig.DeleteTopicEnableProp) def compressionType = getString(KafkaConfig.CompressionTypeProp) diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 021218156681e..d7ca65658f696 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -90,6 +90,7 @@ object KafkaServer { .timeWindow(kafkaConfig.metricSampleWindowMs, TimeUnit.MILLISECONDS) } + val MIN_INCREMENTAL_FETCH_SESSION_EVICTION_MS: Long = 120000 } /** @@ -282,10 +283,14 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP authZ } + val fetchManager = new FetchManager(Time.SYSTEM, + new FetchSessionCache(config.maxIncrementalFetchSessionCacheSlots, + KafkaServer.MIN_INCREMENTAL_FETCH_SESSION_EVICTION_MS)) + /* start processing requests */ apis = new KafkaApis(socketServer.requestChannel, replicaManager, adminManager, groupCoordinator, transactionCoordinator, kafkaController, zkClient, config.brokerId, config, metadataCache, metrics, authorizer, quotaManagers, - brokerTopicStats, clusterId, time, tokenManager) + fetchManager, brokerTopicStats, clusterId, time, tokenManager) requestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.requestChannel, apis, time, config.numIoThreads) diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala index da94c4a8f2163..8344d5beb349a 100644 --- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala @@ -26,6 +26,7 @@ import kafka.log.LogConfig import kafka.server.ReplicaFetcherThread._ import kafka.server.epoch.LeaderEpochCache import kafka.zk.AdminZkClient +import org.apache.kafka.clients.FetchSessionHandler import org.apache.kafka.common.requests.EpochEndOffset._ import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.KafkaStorageException @@ -35,6 +36,7 @@ import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.{EpochEndOffset, FetchResponse, ListOffsetRequest, ListOffsetResponse, OffsetsForLeaderEpochRequest, OffsetsForLeaderEpochResponse, FetchRequest => JFetchRequest} import org.apache.kafka.common.utils.{LogContext, Time} + import scala.collection.JavaConverters._ import scala.collection.{Map, mutable} @@ -65,17 +67,20 @@ class ReplicaFetcherThread(name: String, new ReplicaFetcherBlockingSend(sourceBroker, brokerConfig, metrics, time, fetcherId, s"broker-$replicaId-fetcher-$fetcherId", logContext)) private val fetchRequestVersion: Short = - if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV1) 5 + if (brokerConfig.interBrokerProtocolVersion >= KAFKA_1_1_IV0) 7 + else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV1) 5 else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV0) 4 else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_1_IV1) 3 else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_0_IV0) 2 else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_9_0) 1 else 0 + private val fetchMetadataSupported = brokerConfig.interBrokerProtocolVersion >= KAFKA_1_1_IV0 private val maxWait = brokerConfig.replicaFetchWaitMaxMs private val minBytes = brokerConfig.replicaFetchMinBytes private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes private val fetchSize = brokerConfig.replicaFetchMaxBytes private val shouldSendLeaderEpochRequest: Boolean = brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV2 + private val fetchSessionHandler = new FetchSessionHandler(logContext, sourceBroker.id) private def epochCacheOpt(tp: TopicPartition): Option[LeaderEpochCache] = replicaMgr.getReplica(tp).map(_.epochs.get) @@ -211,10 +216,20 @@ class ReplicaFetcherThread(name: String, } protected def fetch(fetchRequest: FetchRequest): Seq[(TopicPartition, PartitionData)] = { - val clientResponse = leaderEndpoint.sendRequest(fetchRequest.underlying) - val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse] - fetchResponse.responseData.asScala.toSeq.map { case (key, value) => - key -> new PartitionData(value) + try { + val clientResponse = leaderEndpoint.sendRequest(fetchRequest.underlying) + val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse] + if (!fetchSessionHandler.handleResponse(fetchResponse)) { + Nil + } else { + fetchResponse.responseData.asScala.toSeq.map { case (key, value) => + key -> new PartitionData(value) + } + } + } catch { + case t: Throwable => + fetchSessionHandler.handleError(t) + throw t } } @@ -240,15 +255,16 @@ class ReplicaFetcherThread(name: String, } override def buildFetchRequest(partitionMap: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[FetchRequest] = { - val requestMap = new util.LinkedHashMap[TopicPartition, JFetchRequest.PartitionData] val partitionsWithError = mutable.Set[TopicPartition]() + val builder = fetchSessionHandler.newBuilder() partitionMap.foreach { case (topicPartition, partitionFetchState) => // We will not include a replica in the fetch request if it should be throttled. if (partitionFetchState.isReadyForFetch && !shouldFollowerThrottle(quota, topicPartition)) { try { val logStartOffset = replicaMgr.getReplicaOrException(topicPartition).logStartOffset - requestMap.put(topicPartition, new JFetchRequest.PartitionData(partitionFetchState.fetchOffset, logStartOffset, fetchSize)) + builder.add(topicPartition, new JFetchRequest.PartitionData( + partitionFetchState.fetchOffset, logStartOffset, fetchSize)) } catch { case _: KafkaStorageException => // The replica has already been marked offline due to log directory failure and the original failure should have already been logged. @@ -258,9 +274,15 @@ class ReplicaFetcherThread(name: String, } } - val requestBuilder = JFetchRequest.Builder.forReplica(fetchRequestVersion, replicaId, maxWait, minBytes, requestMap) - .setMaxBytes(maxBytes) - ResultWithPartitions(new FetchRequest(requestBuilder), partitionsWithError) + val fetchData = builder.build() + val requestBuilder = JFetchRequest.Builder. + forReplica(fetchRequestVersion, replicaId, maxWait, minBytes, fetchData.toSend()) + .setMaxBytes(maxBytes) + .toForget(fetchData.toForget) + if (fetchMetadataSupported) { + requestBuilder.metadata(fetchData.metadata()) + } + ResultWithPartitions(new FetchRequest(fetchData.sessionPartitions(), requestBuilder), partitionsWithError) } /** @@ -365,10 +387,12 @@ class ReplicaFetcherThread(name: String, object ReplicaFetcherThread { - private[server] class FetchRequest(val underlying: JFetchRequest.Builder) extends AbstractFetcherThread.FetchRequest { - def isEmpty: Boolean = underlying.fetchData.isEmpty + private[server] class FetchRequest(val sessionParts: util.Map[TopicPartition, JFetchRequest.PartitionData], + val underlying: JFetchRequest.Builder) + extends AbstractFetcherThread.FetchRequest { def offset(topicPartition: TopicPartition): Long = - underlying.fetchData.asScala(topicPartition).fetchOffset + sessionParts.get(topicPartition).fetchOffset + override def isEmpty = sessionParts.isEmpty && underlying.toForget().isEmpty override def toString = underlying.toString } diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala index 9090fdac3ac90..f2b3552feaf5b 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala @@ -28,6 +28,7 @@ import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record.{Record, RecordBatch} import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} +import org.apache.kafka.common.requests.{FetchMetadata => JFetchMetadata} import org.apache.kafka.common.serialization.{ByteArraySerializer, StringSerializer} import org.junit.Assert._ import org.junit.Test @@ -294,6 +295,60 @@ class FetchRequestTest extends BaseRequestTest { expectedMagic = RecordBatch.MAGIC_VALUE_V2) } + /** + * Test that when an incremental fetch session contains partitions with an error, + * those partitions are returned in all incremental fetch requests. + */ + @Test + def testCreateIncrementalFetchWithPartitionsInError(): Unit = { + def createFetchRequest(topicPartitions: Seq[TopicPartition], + metadata: JFetchMetadata, + toForget: Seq[TopicPartition]): FetchRequest = + FetchRequest.Builder.forConsumer(Int.MaxValue, 0, + createPartitionMap(Integer.MAX_VALUE, topicPartitions, Map.empty)) + .toForget(toForget.asJava) + .metadata(metadata) + .build() + val foo0 = new TopicPartition("foo", 0) + val foo1 = new TopicPartition("foo", 1) + createTopic("foo", Map(0 -> List(0, 1), 1 -> List(0, 2))) + val bar0 = new TopicPartition("bar", 0) + val req1 = createFetchRequest(List(foo0, foo1, bar0), JFetchMetadata.INITIAL, Nil) + val resp1 = sendFetchRequest(0, req1) + assertEquals(Errors.NONE, resp1.error()) + assertTrue("Expected the broker to create a new incremental fetch session", resp1.sessionId() > 0) + debug(s"Test created an incremental fetch session ${resp1.sessionId}") + assertTrue(resp1.responseData().containsKey(foo0)) + assertTrue(resp1.responseData().containsKey(foo1)) + assertTrue(resp1.responseData().containsKey(bar0)) + assertEquals(Errors.NONE, resp1.responseData().get(foo0).error) + assertEquals(Errors.NONE, resp1.responseData().get(foo1).error) + assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, resp1.responseData().get(bar0).error) + val req2 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 1), Nil) + val resp2 = sendFetchRequest(0, req2) + assertEquals(Errors.NONE, resp2.error()) + assertEquals("Expected the broker to continue the incremental fetch session", + resp1.sessionId(), resp2.sessionId()) + assertFalse(resp2.responseData().containsKey(foo0)) + assertFalse(resp2.responseData().containsKey(foo1)) + assertTrue(resp2.responseData().containsKey(bar0)) + assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, resp2.responseData().get(bar0).error) + createTopic("bar", Map(0 -> List(0, 1))) + val req3 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 2), Nil) + val resp3 = sendFetchRequest(0, req3) + assertEquals(Errors.NONE, resp3.error()) + assertFalse(resp3.responseData().containsKey(foo0)) + assertFalse(resp3.responseData().containsKey(foo1)) + assertTrue(resp3.responseData().containsKey(bar0)) + assertEquals(Errors.NONE, resp3.responseData().get(bar0).error) + val req4 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 3), Nil) + val resp4 = sendFetchRequest(0, req4) + assertEquals(Errors.NONE, resp4.error()) + assertFalse(resp4.responseData().containsKey(foo0)) + assertFalse(resp4.responseData().containsKey(foo1)) + assertFalse(resp4.responseData().containsKey(bar0)) + } + private def records(partitionData: FetchResponse.PartitionData): Seq[Record] = { partitionData.records.records.asScala.toIndexedSeq } diff --git a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala new file mode 100755 index 0000000000000..3320b63938e94 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala @@ -0,0 +1,312 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package kafka.server + +import java.util +import java.util.Collections + +import kafka.utils.MockTime +import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.requests.FetchMetadata.{FINAL_EPOCH, INITIAL_EPOCH, INVALID_SESSION_ID} +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} +import org.junit.{Rule, Test} +import org.junit.Assert._ +import org.junit.rules.Timeout + +class FetchSessionTest { + @Rule + def globalTimeout = Timeout.millis(120000) + + @Test + def testNewSessionId(): Unit = { + val cache = new FetchSessionCache(3, 100) + for (i <- 0 to 10000) { + val id = cache.newSessionId() + assertTrue(id > 0) + } + } + + def assertCacheContains(cache: FetchSessionCache, sessionIds: Int*) = { + var i = 0 + for (sessionId <- sessionIds) { + i = i + 1 + assertTrue("Missing session " + i + " out of " + sessionIds.size + "(" + sessionId + ")", + cache.get(sessionId).isDefined) + } + assertEquals(sessionIds.size, cache.size()) + } + + private def dummyCreate(size: Int)() = { + val cacheMap = new FetchSession.CACHE_MAP(size) + for (i <- 0 to (size - 1)) { + cacheMap.add(new CachedPartition("test", i)) + } + cacheMap + } + + @Test + def testSessionCache(): Unit = { + val cache = new FetchSessionCache(3, 100) + assertEquals(0, cache.size()) + val id1 = cache.maybeCreateSession(0, false, 10, dummyCreate(10)) + val id2 = cache.maybeCreateSession(10, false, 20, dummyCreate(20)) + val id3 = cache.maybeCreateSession(20, false, 30, dummyCreate(30)) + assertEquals(INVALID_SESSION_ID, cache.maybeCreateSession(30, false, 40, dummyCreate(40))) + assertEquals(INVALID_SESSION_ID, cache.maybeCreateSession(40, false, 5, dummyCreate(5))) + assertCacheContains(cache, id1, id2, id3) + cache.touch(cache.get(id1).get, 200) + val id4 = cache.maybeCreateSession(210, false, 11, dummyCreate(11)) + assertCacheContains(cache, id1, id3, id4) + cache.touch(cache.get(id1).get, 400) + cache.touch(cache.get(id3).get, 390) + cache.touch(cache.get(id4).get, 400) + val id5 = cache.maybeCreateSession(410, false, 50, dummyCreate(50)) + assertCacheContains(cache, id3, id4, id5) + assertEquals(INVALID_SESSION_ID, cache.maybeCreateSession(410, false, 5, dummyCreate(5))) + val id6 = cache.maybeCreateSession(410, true, 5, dummyCreate(5)) + assertCacheContains(cache, id3, id5, id6) + } + + @Test + def testResizeCachedSessions(): Unit = { + val cache = new FetchSessionCache(2, 100) + assertEquals(0, cache.totalPartitions()) + assertEquals(0, cache.size()) + assertEquals(0, cache.evictionsMeter.count()) + val id1 = cache.maybeCreateSession(0, false, 2, dummyCreate(2)) + assertTrue(id1 > 0) + assertCacheContains(cache, id1) + val session1 = cache.get(id1).get + assertEquals(2, session1.size()) + assertEquals(2, cache.totalPartitions()) + assertEquals(1, cache.size()) + assertEquals(0, cache.evictionsMeter.count()) + val id2 = cache.maybeCreateSession(0, false, 4, dummyCreate(4)) + val session2 = cache.get(id2).get + assertTrue(id2 > 0) + assertCacheContains(cache, id1, id2) + assertEquals(6, cache.totalPartitions()) + assertEquals(2, cache.size()) + assertEquals(0, cache.evictionsMeter.count()) + cache.touch(session1, 200) + cache.touch(session2, 200) + val id3 = cache.maybeCreateSession(200, false, 5, dummyCreate(5)) + assertTrue(id3 > 0) + assertCacheContains(cache, id2, id3) + assertEquals(9, cache.totalPartitions()) + assertEquals(2, cache.size()) + assertEquals(1, cache.evictionsMeter.count()) + cache.remove(id3) + assertCacheContains(cache, id2) + assertEquals(1, cache.size()) + assertEquals(1, cache.evictionsMeter.count()) + assertEquals(4, cache.totalPartitions()) + val iter = session2.partitionMap.iterator() + iter.next() + iter.remove() + assertEquals(3, session2.size()) + assertEquals(4, session2.cachedSize) + cache.touch(session2, session2.lastUsedMs) + assertEquals(3, cache.totalPartitions()) + } + + val EMPTY_PART_LIST = Collections.unmodifiableList(new util.ArrayList[TopicPartition]()) + + @Test + def testFetchRequests(): Unit = { + val time = new MockTime() + val cache = new FetchSessionCache(10, 1000) + val fetchManager = new FetchManager(time, cache) + + // Verify that SESSIONLESS requests get a SessionlessFetchContext + val context = fetchManager.newContext(JFetchMetadata.LEGACY, + new util.HashMap[TopicPartition, FetchRequest.PartitionData](), EMPTY_PART_LIST, true) + assertEquals(classOf[SessionlessFetchContext], context.getClass) + + // Create a new fetch session with a FULL fetch request + val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + reqData2.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100)) + reqData2.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100)) + val context2 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData2, EMPTY_PART_LIST, false) + assertEquals(classOf[FullFetchContext], context2.getClass) + val reqData2Iter = reqData2.entrySet().iterator() + context2.foreachPartition((topicPart, data) => { + val entry = reqData2Iter.next() + assertEquals(entry.getKey, topicPart) + assertEquals(entry.getValue, data) + }) + assertEquals(0, context2.getFetchOffset(new TopicPartition("foo", 0)).get) + assertEquals(10, context2.getFetchOffset(new TopicPartition("foo", 1)).get) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + respData2.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( + Errors.NONE, 100, 100, 100, null, null)) + respData2.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( + Errors.NONE, 10, 10, 10, null, null)) + val resp2 = context2.updateAndGenerateResponseData(respData2) + assertEquals(Errors.NONE, resp2.error()) + assertTrue(resp2.sessionId() != INVALID_SESSION_ID) + assertEquals(respData2, resp2.responseData()) + + // Test trying to create a new session with an invalid epoch + val context3 = fetchManager.newContext( + new JFetchMetadata(resp2.sessionId(), 5), reqData2, EMPTY_PART_LIST, false) + assertEquals(classOf[SessionErrorContext], context3.getClass) + assertEquals(Errors.INVALID_FETCH_SESSION_EPOCH, + context3.updateAndGenerateResponseData(respData2).error()) + + // Test trying to create a new session with a non-existent session id + val context4 = fetchManager.newContext( + new JFetchMetadata(resp2.sessionId() + 1, 1), reqData2, EMPTY_PART_LIST, false) + assertEquals(classOf[SessionErrorContext], context4.getClass) + assertEquals(Errors.FETCH_SESSION_ID_NOT_FOUND, + context4.updateAndGenerateResponseData(respData2).error()) + + // Continue the first fetch session we created. + val reqData5 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + val context5 = fetchManager.newContext( + new JFetchMetadata(resp2.sessionId(), 1), reqData5, EMPTY_PART_LIST, false) + assertEquals(classOf[IncrementalFetchContext], context5.getClass) + val reqData5Iter = reqData2.entrySet().iterator() + context5.foreachPartition((topicPart, data) => { + val entry = reqData5Iter.next() + assertEquals(entry.getKey, topicPart) + assertEquals(entry.getValue, data) + }) + assertEquals(10, context5.getFetchOffset(new TopicPartition("foo", 1)).get) + val resp5 = context5.updateAndGenerateResponseData(respData2) + assertEquals(Errors.NONE, resp5.error()) + assertEquals(resp2.sessionId(), resp5.sessionId()) + assertEquals(0, resp5.responseData().size()) + + // Test setting an invalid fetch session epoch. + val context6 = fetchManager.newContext( + new JFetchMetadata(resp2.sessionId(), 5), reqData2, EMPTY_PART_LIST, false) + assertEquals(classOf[SessionErrorContext], context6.getClass) + assertEquals(Errors.INVALID_FETCH_SESSION_EPOCH, + context6.updateAndGenerateResponseData(respData2).error()) + + // Close the incremental fetch session. + var prevSessionId = resp5.sessionId() + var nextSessionId = prevSessionId + do { + val reqData7 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + reqData7.put(new TopicPartition("bar", 0), new FetchRequest.PartitionData(0, 0, 100)) + reqData7.put(new TopicPartition("bar", 1), new FetchRequest.PartitionData(10, 0, 100)) + val context7 = fetchManager.newContext( + new JFetchMetadata(prevSessionId, FINAL_EPOCH), reqData7, EMPTY_PART_LIST, false) + assertEquals(classOf[SessionlessFetchContext], context7.getClass) + assertEquals(0, cache.size()) + val respData7 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + respData7.put(new TopicPartition("bar", 0), + new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) + respData7.put(new TopicPartition("bar", 1), + new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) + val resp7 = context7.updateAndGenerateResponseData(respData7) + assertEquals(Errors.NONE, resp7.error()) + nextSessionId = resp7.sessionId() + } while (nextSessionId == prevSessionId) + } + + @Test + def testIncrementalFetchSession(): Unit = { + val time = new MockTime() + val cache = new FetchSessionCache(10, 1000) + val fetchManager = new FetchManager(time, cache) + + // Create a new fetch session with foo-0 and foo-1 + val reqData1 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + reqData1.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100)) + reqData1.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100)) + val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false) + assertEquals(classOf[FullFetchContext], context1.getClass) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( + Errors.NONE, 100, 100, 100, null, null)) + respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( + Errors.NONE, 10, 10, 10, null, null)) + val resp1 = context1.updateAndGenerateResponseData(respData1) + assertEquals(Errors.NONE, resp1.error()) + assertTrue(resp1.sessionId() != INVALID_SESSION_ID) + assertEquals(2, resp1.responseData().size()) + + // Create an incremental fetch request that removes foo-0 and adds bar-0 + val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + reqData2.put(new TopicPartition("bar", 0), new FetchRequest.PartitionData(15, 0, 0)) + val removed2 = new util.ArrayList[TopicPartition] + removed2.add(new TopicPartition("foo", 0)) + val context2 = fetchManager.newContext( + new JFetchMetadata(resp1.sessionId(), 1), reqData2, removed2, false) + assertEquals(classOf[IncrementalFetchContext], context2.getClass) + val parts2 = Set(new TopicPartition("foo", 1), new TopicPartition("bar", 0)) + val reqData2Iter = parts2.iterator + context2.foreachPartition((topicPart, data) => { + assertEquals(reqData2Iter.next(), topicPart) + }) + assertEquals(None, context2.getFetchOffset(new TopicPartition("foo", 0))) + assertEquals(10, context2.getFetchOffset(new TopicPartition("foo", 1)).get) + assertEquals(15, context2.getFetchOffset(new TopicPartition("bar", 0)).get) + assertEquals(None, context2.getFetchOffset(new TopicPartition("bar", 2))) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + respData2.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( + Errors.NONE, 10, 10, 10, null, null)) + respData2.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData( + Errors.NONE, 10, 10, 10, null, null)) + val resp2 = context2.updateAndGenerateResponseData(respData2) + assertEquals(Errors.NONE, resp2.error()) + assertEquals(1, resp2.responseData().size()) + assertTrue(resp2.sessionId() > 0) + } + + @Test + def testZeroSizeFetchSession(): Unit = { + val time = new MockTime() + val cache = new FetchSessionCache(10, 1000) + val fetchManager = new FetchManager(time, cache) + + // Create a new fetch session with foo-0 and foo-1 + val reqData1 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + reqData1.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100)) + reqData1.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100)) + val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false) + assertEquals(classOf[FullFetchContext], context1.getClass) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( + Errors.NONE, 100, 100, 100, null, null)) + respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( + Errors.NONE, 10, 10, 10, null, null)) + val resp1 = context1.updateAndGenerateResponseData(respData1) + assertEquals(Errors.NONE, resp1.error()) + assertTrue(resp1.sessionId() != INVALID_SESSION_ID) + assertEquals(2, resp1.responseData().size()) + + // Create an incremental fetch request that removes foo-0 and foo-1 + // Verify that the previous fetch session was closed. + val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] + val removed2 = new util.ArrayList[TopicPartition] + removed2.add(new TopicPartition("foo", 0)) + removed2.add(new TopicPartition("foo", 1)) + val context2 = fetchManager.newContext( + new JFetchMetadata(resp1.sessionId(), 1), reqData2, removed2, false) + assertEquals(classOf[SessionlessFetchContext], context2.getClass) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData] + val resp2 = context2.updateAndGenerateResponseData(respData2) + assertEquals(INVALID_SESSION_ID, resp2.sessionId()) + assertTrue(resp2.responseData().isEmpty) + assertEquals(0, cache.size()) + } +} diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 8e907d930e9c7..5de978cd0beb8 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -69,6 +69,7 @@ class KafkaApisTest { private val clientRequestQuotaManager = EasyMock.createNiceMock(classOf[ClientRequestQuotaManager]) private val replicaQuotaManager = EasyMock.createNiceMock(classOf[ReplicationQuotaManager]) private val quotas = QuotaManagers(clientQuotaManager, clientQuotaManager, clientRequestQuotaManager, replicaQuotaManager, replicaQuotaManager, replicaQuotaManager) + private val fetchManager = EasyMock.createNiceMock(classOf[FetchManager]) private val brokerTopicStats = new BrokerTopicStats private val clusterId = "clusterId" private val time = new MockTime @@ -96,6 +97,7 @@ class KafkaApisTest { metrics, authorizer, quotas, + fetchManager, brokerTopicStats, clusterId, time, diff --git a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala index 0692afb6591c1..1f5bec1cf8220 100644 --- a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala +++ b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala @@ -20,10 +20,10 @@ import kafka.cluster.BrokerEndPoint import kafka.server.BlockingSend import org.apache.kafka.clients.{ClientRequest, ClientResponse, MockClient} import org.apache.kafka.common.{Node, TopicPartition} -import org.apache.kafka.common.protocol.ApiKeys +import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.AbstractRequest.Builder import org.apache.kafka.common.requests.FetchResponse.PartitionData -import org.apache.kafka.common.requests.{AbstractRequest, EpochEndOffset, FetchResponse, OffsetsForLeaderEpochResponse} +import org.apache.kafka.common.requests.{AbstractRequest, EpochEndOffset, FetchResponse, OffsetsForLeaderEpochResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.{SystemTime, Time} /** @@ -54,7 +54,8 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc case ApiKeys.FETCH => fetchCount += 1 - new FetchResponse(new java.util.LinkedHashMap[TopicPartition, PartitionData], 0) + new FetchResponse(Errors.NONE, new java.util.LinkedHashMap[TopicPartition, PartitionData], 0, + JFetchMetadata.INVALID_SESSION_ID) case _ => throw new UnsupportedOperationException From ed971fd4341f1643cba38684a0be3f9362e9394c Mon Sep 17 00:00:00 2001 From: Manikumar Reddy Date: Mon, 5 Feb 2018 11:07:18 -0800 Subject: [PATCH 0004/1847] KAFKA-6452; Add documentation for delegation token authentication Author: Manikumar Reddy Reviewers: Jun Rao Closes #4490 from omkreddy/KAFKA-6452-TOKEN-DOCS --- docs/security.html | 102 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/docs/security.html b/docs/security.html index 4e401aec186c4..3e3c818571aba 100644 --- a/docs/security.html +++ b/docs/security.html @@ -661,6 +661,108 @@

7.3 Authentication using SASL + +

  • Authentication using Delegation Tokens

    +

    Delegation token based authentication is a lightweight authentication mechanism to complement existing SASL/SSL + methods. Delegation tokens are shared secrets between kafka brokers and clients. Delegation tokens will help processing + frameworks to distribute the workload to available workers in a secure environment without the added cost of distributing + Kerberos TGT/keytabs or keystores when 2-way SSL is used. See KIP-48 + for more details.

    + +

    Typical steps for delegation token usage are:

    +
      +
    1. User authenticates with the Kafka cluster via SASL or SSL, and obtains a delegation token. This can be done + using AdminClient APIs or using kafka-delegation-token.sh script.
      2. +
    2. User securely passes the delegation token to Kafka clients for authenticating with the Kafka cluster.
      3. +
    3. Token owner/renewer can renew/expire the delegation tokens.
      4. +
    + +
      +
    1. Token Management
      +

      A master key/secret is used to generate and verify delegation tokens. This is supplied using config + option delegation.token.master.key. Same secret key must be configured across all the brokers. + If the secret is not set or set to empty string, brokers will disable the delegation token authentication.

      + +

      In current implementation, token details are stored in Zookeeper and is suitable for use in Kafka installations where + Zookeeper is on a private network. Also currently, master key/secret is stored as plain text in server.properties + config file. We intend to make these configurable in a future Kafka release.

      + +

      A token has a current life, and a maximum renewable life. By default, tokens must be renewed once every 24 hours + for up to 7 days. These can be configured using delegation.token.expiry.time.ms + and delegation.token.max.lifetime.ms config options.

      + +

      Tokens can also be cancelled explicitly. If a token is not renewed by the token’s expiration time or if token + is beyond the max life time, it will be deleted from all broker caches as well as from zookeeper.

      +
      2. + +
    2. Creating Delegation Tokens
      +

      Tokens can be created by using AdminClient APIs or using kafka-delegation-token.sh script. + Delegation token requests (create/renew/expire/describe) should be issued only on SASL or SSL authenticated channels. + Tokens can not be requests if the initial authentication is done through delegation token. + kafka-delegation-token.sh script examples are given below.

      +

      Create a delegation token: + 

      +    > bin/kafka-delegation-tokens.sh --bootstrap-server localhost:9092 --create   --max-life-time-period -1 --command-config client.properties --renewer-principal User:user1
+
      +

      Renew a delegation token: + 

      +    > bin/kafka-delegation-tokens.sh --bootstrap-server localhost:9092 --renew    --renew-time-period -1 --command-config client.properties --hmac ABCDEFGHIJK
+
      +

      Expire a delegation token: + 

      +    > bin/kafka-delegation-tokens.sh --bootstrap-server localhost:9092 --expire   --expiry-time-period -1   --command-config client.properties  --hmac ABCDEFGHIJK
+
      +

      Existing tokens can be described using the --describe option: + 

      +    > bin/kafka-delegation-tokens.sh --bootstrap-server localhost:9092 --describe --command-config client.properties  --owner-principal User:user1
+
      +
      3. +
    3. Token Authentication
      +

      Delegation token authentication piggybacks on the current SASL/SCRAM authentication mechanism. We must enable + SASL/SCRAM mechanism on Kafka cluster as described in here.

      + +

      Configuring Kafka Clients:

      +
        +
      1. Configure the JAAS configuration property for each client in producer.properties or consumer.properties. + The login module describes how the clients like producer and consumer can connect to the Kafka Broker. + The following is an example configuration for a client for the token authentication: + 
        +   sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \
+        username="tokenID123" \
+        password="lAYYSFmLs4bTjf+lTZ1LCHR/ZZFNA==" \
+        tokenauth="true";
        + +

        The options username and password are used by clients to configure the token id and + token HMAC. And the option tokenauth is used to indicate the server about token authentication. + In this example, clients connect to the broker using token id: tokenID123. Different clients within a + JVM may connect using different tokens by specifying different token details in sasl.jaas.config.

        + +

        JAAS configuration for clients may alternatively be specified as a JVM parameter similar to brokers + as described here. Clients use the login section named + KafkaClient. This option allows only one user for all client connections from a JVM.

        2. +
      +
      4. + +
    4. Procedure to manually rotate the secret:
      +

      We require a re-deployment when the secret needs to be rotated. During this process, already connected clients + will continue to work. But any new connection requests and renew/expire requests with old tokens can fail. Steps are given below.

      + +
        +
      1. Expire all existing tokens.
        2. +
      2. Rotate the secret by rolling upgrade, and
        3. +
      3. Generate new tokens
        4. +
      +

      We intend to automate this in a future Kafka release.

      +
      5. + +
    5. Notes on Delegation Tokens
      +
        +
      • Currently, we only allow a user to create delegation token for that user only. Owner/Renewers can renew or expire tokens. + Owner/renewers can always describe their own tokens. To describe others tokens, we need to add DESCRIBE permission on Token Resource.
        • +
      +
      6. +
    +

    • 7.4 Authorization and ACLs

    From 447c64fffb72fdb82e2f3c670e891ae1d00a8ec8 Mon Sep 17 00:00:00 2001 From: Randall Hauch Date: Mon, 5 Feb 2018 12:13:26 -0800 Subject: [PATCH 0005/1847] KAFKA-5987: Maintain order of metric tags in generated documentation The `MetricNameTemplate` is changed to used a `LinkedHashSet` to maintain the same order of the tags that are passed in. This tag order is then maintained when `Metrics.toHtmlTable` generates the MBean names for each of the metrics. The `SenderMetricsRegistry` and `FetcherMetricsRegistry` both contain templates used in the producer and consumer, respectively, and these were changed to use a `LinkedHashSet` to maintain the order of the tags. Before this change, the generated HTML documentation might use MBean names like the following and order them: ``` kafka.connect:type=sink-task-metrics,connector={connector},partition={partition},task={task},topic={topic} kafka.connect:type=sink-task-metrics,connector={connector},task={task} ``` However, after this change, the documentation would use the following order: ``` kafka.connect:type=sink-task-metrics,connector={connector},task={task} kafka.connect:type=sink-task-metrics,connector={connector},task={task},topic={topic},partition={partition} ``` This is more readable as the code that is creating the templates has control over the order of the tags. Note that JMX MBean names use ObjectName that does not maintain order of the properties (tags), so this change should have no impact on the actual JMX MBean names used in the metrics. cc wushujames Author: Randall Hauch Reviewers: James Cheng , Ewen Cheslack-Postava Closes #3985 from rhauch/kafka-5987 --- .../internals/FetcherMetricsRegistry.java | 3 +- .../internals/SenderMetricsRegistry.java | 8 +-- .../kafka/common/MetricNameTemplate.java | 58 +++++++++++++++---- .../apache/kafka/common/metrics/Metrics.java | 14 ++++- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java index f15ac062abf96..301363a638b1f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -102,7 +103,7 @@ public FetcherMetricsRegistry(Set tags, String metricGrpPrefix) { "The maximum throttle time in ms", tags); /***** Topic level *****/ - Set topicTags = new HashSet<>(tags); + Set topicTags = new LinkedHashSet<>(tags); topicTags.add("topic"); this.topicFetchSizeAvg = new MetricNameTemplate("fetch-size-avg", groupName, diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/SenderMetricsRegistry.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/SenderMetricsRegistry.java index 21dbca618303d..b01236f6218af 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/SenderMetricsRegistry.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/SenderMetricsRegistry.java @@ -17,7 +17,7 @@ package org.apache.kafka.clients.producer.internals; import java.util.ArrayList; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -70,12 +70,12 @@ public class SenderMetricsRegistry { private final Metrics metrics; private final Set tags; - private final HashSet topicTags; + private final LinkedHashSet topicTags; public SenderMetricsRegistry(Metrics metrics) { this.metrics = metrics; this.tags = this.metrics.config().tags().keySet(); - this.allTemplates = new ArrayList(); + this.allTemplates = new ArrayList<>(); /***** Client level *****/ @@ -126,7 +126,7 @@ public SenderMetricsRegistry(Metrics metrics) { "The maximum time in ms a request was throttled by a broker"); /***** Topic level *****/ - this.topicTags = new HashSet(tags); + this.topicTags = new LinkedHashSet<>(tags); this.topicTags.add("topic"); // We can't create the MetricName up front for these, because we don't know the topic name yet. diff --git a/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java b/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java index e3ea9950ef159..1b1de71037d1f 100644 --- a/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java +++ b/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.common; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; @@ -26,27 +26,45 @@ * A template for a MetricName. It contains a name, group, and description, as * well as all the tags that will be used to create the mBean name. Tag values * are omitted from the template, but are filled in at runtime with their - * specified values. + * specified values. The order of the tags is maintained, if an ordered set + * is provided, so that the mBean names can be compared and sorted lexicographically. */ public class MetricNameTemplate { private final String name; private final String group; private final String description; - private Set tags; + private LinkedHashSet tags; - public MetricNameTemplate(String name, String group, String description, Set tags) { + /** + * Create a new template. Note that the order of the tags will be preserved if the supplied + * {@code tagsNames} set has an order. + * + * @param name the name of the metric; may not be null + * @param group the name of the group; may not be null + * @param description the description of the metric; may not be null + * @param tagsNames the set of metric tag names, which can/should be a set that maintains order; may not be null + */ + public MetricNameTemplate(String name, String group, String description, Set tagsNames) { this.name = Utils.notNull(name); this.group = Utils.notNull(group); this.description = Utils.notNull(description); - this.tags = Utils.notNull(tags); + this.tags = new LinkedHashSet<>(Utils.notNull(tagsNames)); } - - public MetricNameTemplate(String name, String group, String description, String... keys) { - this(name, group, description, getTags(keys)); + + /** + * Create a new template. Note that the order of the tags will be preserved. + * + * @param name the name of the metric; may not be null + * @param group the name of the group; may not be null + * @param description the description of the metric; may not be null + * @param tagsNames the names of the metric tags in the preferred order; none of the tag names should be null + */ + public MetricNameTemplate(String name, String group, String description, String... tagsNames) { + this(name, group, description, getTags(tagsNames)); } - private static Set getTags(String... keys) { - Set tags = new HashSet(); + private static LinkedHashSet getTags(String... keys) { + LinkedHashSet tags = new LinkedHashSet<>(); for (int i = 0; i < keys.length; i++) tags.add(keys[i]); @@ -54,18 +72,38 @@ private static Set getTags(String... keys) { return tags; } + /** + * Get the name of the metric. + * + * @return the metric name; never null + */ public String name() { return this.name; } + /** + * Get the name of the group. + * + * @return the group name; never null + */ public String group() { return this.group; } + /** + * Get the description of the metric. + * + * @return the metric description; never null + */ public String description() { return this.description; } + /** + * Get the set of tag names for the metric. + * + * @return the ordered set of tag names; never null but possibly empty + */ public Set tags() { return tags; } diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java index 7f9fb9dad5acf..8868ee7ac48f5 100644 --- a/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java +++ b/clients/src/main/java/org/apache/kafka/common/metrics/Metrics.java @@ -232,12 +232,22 @@ private static Map getTags(String... keyValue) { return tags; } - public static String toHtmlTable(String domain, List allMetrics) { + /** + * Use the specified domain and metric name templates to generate an HTML table documenting the metrics. A separate table section + * will be generated for each of the MBeans and the associated attributes. The MBean names are lexicographically sorted to + * determine the order of these sections. This order is therefore dependent upon the order of the + * tags in each {@link MetricNameTemplate}. + * + * @param domain the domain or prefix for the JMX MBean names; may not be null + * @param allMetrics the collection of all {@link MetricNameTemplate} instances each describing one metric; may not be null + * @return the string containing the HTML table; never null + */ + public static String toHtmlTable(String domain, Iterable allMetrics) { Map> beansAndAttributes = new TreeMap>(); try (Metrics metrics = new Metrics()) { for (MetricNameTemplate template : allMetrics) { - Map tags = new TreeMap(); + Map tags = new LinkedHashMap<>(); for (String s : template.tags()) { tags.put(s, "{" + s + "}"); } From 2d4fb785409056b7abfdfe6082f2862a138acf74 Mon Sep 17 00:00:00 2001 From: Ewen Cheslack-Postava Date: Mon, 5 Feb 2018 13:42:04 -0800 Subject: [PATCH 0006/1847] MINOR: Update copyright year in NOTICE Author: Ewen Cheslack-Postava Reviewers: Jason Gustafson , Guozhang Wang Closes #4529 from ewencp/update-NOTICE-year --- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 7a62b7fed60cd..c97f407c13154 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Kafka -Copyright 2017 The Apache Software Foundation. +Copyright 2018 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (http://www.apache.org/). From 332e698ac9c74ce29317021b03a54512c92ac8b3 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 5 Feb 2018 16:35:12 -0800 Subject: [PATCH 0007/1847] MINOR: improve log4j messaging (#4530) Reviewers: Matthias J. Sax , James Cheng --- .../kafka/streams/errors/TaskMigratedException.java | 10 ++++++++++ .../streams/processor/internals/PartitionGroup.java | 2 +- .../processor/internals/RecordCollectorImpl.java | 2 +- .../streams/processor/internals/StreamThread.java | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/errors/TaskMigratedException.java b/streams/src/main/java/org/apache/kafka/streams/errors/TaskMigratedException.java index f2fa5942d8ca7..5a284e48705aa 100644 --- a/streams/src/main/java/org/apache/kafka/streams/errors/TaskMigratedException.java +++ b/streams/src/main/java/org/apache/kafka/streams/errors/TaskMigratedException.java @@ -28,6 +28,8 @@ public class TaskMigratedException extends StreamsException { private final static long serialVersionUID = 1L; + private final Task task; + public TaskMigratedException(final Task task) { this(task, null); } @@ -42,11 +44,19 @@ public TaskMigratedException(final Task task, pos, task.toString("> ")), null); + + this.task = task; } public TaskMigratedException(final Task task, final Throwable throwable) { super(task.toString(), throwable); + + this.task = task; + } + + public Task migratedTask() { + return task; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java index 8ce7dc9948f24..dcaa755ecdc31 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java @@ -153,7 +153,7 @@ int numBuffered(final TopicPartition partition) { final RecordQueue recordQueue = partitionQueues.get(partition); if (recordQueue == null) { - throw new IllegalStateException("Record's partition does not belong to this partition-group."); + throw new IllegalStateException(String.format("Record's partition %s does not belong to this partition-group.", partition)); } return recordQueue.size(); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java index afdadf2dec3fe..286cd817c88be 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java @@ -121,7 +121,7 @@ private void recordSendError( errorLogMessage += PARAMETER_HINT; errorMessage += PARAMETER_HINT; } - log.error(errorLogMessage, key, value, timestamp, topic, exception); + log.error(errorLogMessage, key, value, timestamp, topic, exception.toString()); sendException = new StreamsException( String.format(errorMessage, logPrefix, diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java index cb133c67a0b55..064a293551588 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java @@ -755,9 +755,9 @@ private void runLoop() { try { recordsProcessedBeforeCommit = runOnce(recordsProcessedBeforeCommit); } catch (final TaskMigratedException ignoreAndRejoinGroup) { - log.warn("Detected a task that got migrated to another thread. " + + log.warn("Detected task {} that got migrated to another thread. " + "This implies that this thread missed a rebalance and dropped out of the consumer group. " + - "Trying to rejoin the consumer group now.", ignoreAndRejoinGroup); + "Trying to rejoin the consumer group now. Below is the detailed description of the task:\n{}", ignoreAndRejoinGroup.migratedTask().id(), ignoreAndRejoinGroup.migratedTask().toString(">")); } } } From 65d262cc30315ed47bd17e94c0db672de93d908a Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Tue, 6 Feb 2018 01:50:51 -0800 Subject: [PATCH 0008/1847] KAFKA-6528: Fix transient test failure in testThreadPoolResize (#4526) Add locking to access AbstractFetcherThread#partitionStates during dynamic thread update. Also make testing of thread updates that trigger retries more resilient. Reviewers: Jason Gustafson --- .../kafka/server/AbstractFetcherManager.scala | 4 +- .../kafka/server/AbstractFetcherThread.scala | 6 ++ .../DynamicBrokerReconfigurationTest.scala | 92 ++++++++++++------- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala index 6d88d8de8b26b..312123c3ab69d 100755 --- a/core/src/main/scala/kafka/server/AbstractFetcherManager.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherManager.scala @@ -70,9 +70,7 @@ abstract class AbstractFetcherManager(protected val name: String, clientId: Stri def resizeThreadPool(newSize: Int): Unit = { def migratePartitions(newSize: Int): Unit = { fetcherThreadMap.foreach { case (id, thread) => - val removedPartitions = thread.partitionStates.partitionStates.asScala.map { case state => - state.topicPartition -> new BrokerAndInitialOffset(thread.sourceBroker, state.value.fetchOffset) - }.toMap + val removedPartitions = thread.partitionsAndOffsets removeFetcherForPartitions(removedPartitions.keySet) if (id.fetcherId >= newSize) thread.shutdown() diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala index 925c33095a253..39a70321a6ef0 100755 --- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala @@ -312,6 +312,12 @@ abstract class AbstractFetcherThread(name: String, finally partitionMapLock.unlock() } + private[server] def partitionsAndOffsets: Map[TopicPartition, BrokerAndInitialOffset] = inLock(partitionMapLock) { + partitionStates.partitionStates.asScala.map { case state => + state.topicPartition -> new BrokerAndInitialOffset(sourceBroker, state.value.fetchOffset) + }.toMap + } + } object AbstractFetcherThread { diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala index b7f0ae863a2dc..cb2ac5244a06a 100644 --- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala +++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala @@ -69,6 +69,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet private val servers = new ArrayBuffer[KafkaServer] private val numServers = 3 + private val numPartitions = 10 private val producers = new ArrayBuffer[KafkaProducer[String, String]] private val consumers = new ArrayBuffer[KafkaConsumer[String, String]] private val adminClients = new ArrayBuffer[AdminClient]() @@ -122,7 +123,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet TestUtils.createTopic(zkClient, Topic.GROUP_METADATA_TOPIC_NAME, OffsetConfig.DefaultOffsetsTopicNumPartitions, replicationFactor = numServers, servers, servers.head.groupCoordinator.offsetsTopicConfigs) - TestUtils.createTopic(zkClient, topic, numPartitions = 10, replicationFactor = numServers, servers) + TestUtils.createTopic(zkClient, topic, numPartitions, replicationFactor = numServers, servers) createAdminClient(SecurityProtocol.SSL, SecureInternal) TestMetricsReporter.testReporters.clear() @@ -203,7 +204,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet @Test def testKeyStoreAlter(): Unit = { val topic2 = "testtopic2" - TestUtils.createTopic(zkClient, topic2, numPartitions = 10, replicationFactor = numServers, servers) + TestUtils.createTopic(zkClient, topic2, numPartitions, replicationFactor = numServers, servers) // Start a producer and consumer that work with the current truststore. // This should continue working while changes are made @@ -241,7 +242,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet verifyProduceConsume(producer, consumer, 10, topic2) // Verify that all messages sent with retries=0 while keystores were being altered were consumed - stopAndVerifyProduceConsume(producerThread, consumerThread, mayFailRequests = false) + stopAndVerifyProduceConsume(producerThread, consumerThread) } @Test @@ -282,7 +283,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet verifyThreads("kafka-log-cleaner-thread-", countPerBroker = 2) // Verify that produce/consume worked throughout this test without any retries in producer - stopAndVerifyProduceConsume(producerThread, consumerThread, mayFailRequests = false) + stopAndVerifyProduceConsume(producerThread, consumerThread) } @Test @@ -370,7 +371,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet servers.tail.foreach { server => assertEquals(Defaults.LogIndexSizeMaxBytes, server.config.values.get(KafkaConfig.LogIndexSizeMaxBytesProp)) } // Verify that produce/consume worked throughout this test without any retries in producer - stopAndVerifyProduceConsume(producerThread, consumerThread, mayFailRequests = false) + stopAndVerifyProduceConsume(producerThread, consumerThread) } @Test @@ -418,9 +419,9 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet reconfigureServers(props, perBrokerConfig = false, (propName, newSize.toString)) maybeVerifyThreadPoolSize(propName, newSize, threadPrefix) } - def verifyThreadPoolResize(propName: String, currentSize: => Int, threadPrefix: String, mayFailRequests: Boolean): Unit = { + def verifyThreadPoolResize(propName: String, currentSize: => Int, threadPrefix: String, mayReceiveDuplicates: Boolean): Unit = { maybeVerifyThreadPoolSize(propName, currentSize, threadPrefix) - val numRetries = if (mayFailRequests) 100 else 0 + val numRetries = if (mayReceiveDuplicates) 100 else 0 val (producerThread, consumerThread) = startProduceConsume(numRetries) var threadPoolSize = currentSize (1 to 2).foreach { _ => @@ -429,20 +430,20 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet threadPoolSize = increasePoolSize(propName, threadPoolSize, threadPrefix) Thread.sleep(100) } - stopAndVerifyProduceConsume(producerThread, consumerThread, mayFailRequests) + stopAndVerifyProduceConsume(producerThread, consumerThread, mayReceiveDuplicates) } val config = servers.head.config verifyThreadPoolResize(KafkaConfig.NumIoThreadsProp, config.numIoThreads, - requestHandlerPrefix, mayFailRequests = false) - verifyThreadPoolResize(KafkaConfig.NumNetworkThreadsProp, config.numNetworkThreads, - networkThreadPrefix, mayFailRequests = true) + requestHandlerPrefix, mayReceiveDuplicates = false) verifyThreadPoolResize(KafkaConfig.NumReplicaFetchersProp, config.numReplicaFetchers, - fetcherThreadPrefix, mayFailRequests = false) + fetcherThreadPrefix, mayReceiveDuplicates = false) verifyThreadPoolResize(KafkaConfig.BackgroundThreadsProp, config.backgroundThreads, - "kafka-scheduler-", mayFailRequests = false) + "kafka-scheduler-", mayReceiveDuplicates = false) verifyThreadPoolResize(KafkaConfig.NumRecoveryThreadsPerDataDirProp, config.numRecoveryThreadsPerDataDir, - "", mayFailRequests = false) + "", mayReceiveDuplicates = false) + verifyThreadPoolResize(KafkaConfig.NumNetworkThreadsProp, config.numNetworkThreads, + networkThreadPrefix, mayReceiveDuplicates = true) } @Test @@ -1055,17 +1056,16 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet } private def stopAndVerifyProduceConsume(producerThread: ProducerThread, consumerThread: ConsumerThread, - mayFailRequests: Boolean = false): Unit = { + mayReceiveDuplicates: Boolean = false): Unit = { TestUtils.waitUntilTrue(() => producerThread.sent >= 10, "Messages not sent") producerThread.shutdown() consumerThread.initiateShutdown() consumerThread.awaitShutdown() - if (!mayFailRequests) - assertEquals(producerThread.sent, consumerThread.received) - else { - assertTrue(s"Some messages not received, sent=${producerThread.sent} received=${consumerThread.received}", - consumerThread.received >= producerThread.sent) - } + assertEquals(producerThread.lastSent, consumerThread.lastReceived) + assertEquals(0, consumerThread.missingRecords.size) + if (!mayReceiveDuplicates) + assertFalse("Duplicates not expected", consumerThread.duplicates) + assertFalse("Some messages received out of order", consumerThread.outOfOrder) } private def verifyConnectionFailure(producer: KafkaProducer[String, String]): Future[_] = { @@ -1128,32 +1128,58 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet private class ProducerThread(clientId: String, retries: Int) extends ShutdownableThread(clientId, isInterruptible = false) { private val producer = createProducer(trustStoreFile1, retries, clientId) + val lastSent = new ConcurrentHashMap[Int, Int]() @volatile var sent = 0 override def doWork(): Unit = { - try { - while (isRunning) { - sent += 1 - val record = new ProducerRecord(topic, s"key$sent", s"value$sent") - producer.send(record).get(10, TimeUnit.SECONDS) - } - } finally { - producer.close() - } + try { + while (isRunning) { + val key = sent.toString + val partition = sent % numPartitions + val record = new ProducerRecord(topic, partition, key, s"value$sent") + producer.send(record).get(10, TimeUnit.SECONDS) + lastSent.put(partition, sent) + sent += 1 + } + } finally { + producer.close() } + } } private class ConsumerThread(producerThread: ProducerThread) extends ShutdownableThread("test-consumer", isInterruptible = false) { private val consumer = createConsumer("group1", trustStoreFile1) + val lastReceived = new ConcurrentHashMap[Int, Int]() + val missingRecords = new ConcurrentLinkedQueue[Int]() + @volatile var outOfOrder = false + @volatile var duplicates = false @volatile var lastBatch: ConsumerRecords[String, String] = _ @volatile private var endTimeMs = Long.MaxValue - var received = 0 + @volatile var received = 0 override def doWork(): Unit = { try { - while (isRunning || (received < producerThread.sent && System.currentTimeMillis < endTimeMs)) { + while (isRunning || (lastReceived != producerThread.lastSent && System.currentTimeMillis < endTimeMs)) { val records = consumer.poll(50) received += records.count - if (!records.isEmpty) + if (!records.isEmpty) { lastBatch = records + records.partitions.asScala.foreach { tp => + val partition = tp.partition + records.records(tp).asScala.map(_.key.toInt).foreach { key => + val prevKey = lastReceived.asScala.get(partition).getOrElse(partition - numPartitions) + val expectedKey = prevKey + numPartitions + if (key < prevKey) + outOfOrder = true + else if (key == prevKey) + duplicates = true + else { + for (i <- expectedKey until key by numPartitions) + missingRecords.add(expectedKey) + } + lastReceived.put(partition, key) + missingRecords.remove(key) + } + } + } } } finally { consumer.close() From 7d70a427b2689c0a1de4345db4b04ea557b4c27b Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Tue, 6 Feb 2018 09:18:37 -0800 Subject: [PATCH 0009/1847] KAFKA-6504: Ensure uniqueness of connect task metric sensor creation (#4514) This change ensures that when sensors are created, they are unique to the metric group associated with the task that created them. Previously the sensors were being shared between task metric groups, causing incorrect metrics. Reviewers: Randall Hauch , Jason Gustafson --- .../kafka/connect/runtime/WorkerSinkTask.java | 16 ++--- .../connect/runtime/WorkerSourceTask.java | 2 +- .../connect/runtime/WorkerSinkTaskTest.java | 65 +++++++++++++++++++ .../connect/runtime/WorkerSourceTaskTest.java | 34 ++++++++++ 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java index 5aeb851791ff2..2995a4e813aa8 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java @@ -684,34 +684,34 @@ public SinkTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) { // prevent collisions by removing any previously created metrics in this group. metricGroup.close(); - sinkRecordRead = metricGroup.metrics().sensor("sink-record-read"); + sinkRecordRead = metricGroup.sensor("sink-record-read"); sinkRecordRead.add(metricGroup.metricName(registry.sinkRecordReadRate), new Rate()); sinkRecordRead.add(metricGroup.metricName(registry.sinkRecordReadTotal), new Total()); - sinkRecordSend = metricGroup.metrics().sensor("sink-record-send"); + sinkRecordSend = metricGroup.sensor("sink-record-send"); sinkRecordSend.add(metricGroup.metricName(registry.sinkRecordSendRate), new Rate()); sinkRecordSend.add(metricGroup.metricName(registry.sinkRecordSendTotal), new Total()); - sinkRecordActiveCount = metricGroup.metrics().sensor("sink-record-active-count"); + sinkRecordActiveCount = metricGroup.sensor("sink-record-active-count"); sinkRecordActiveCount.add(metricGroup.metricName(registry.sinkRecordActiveCount), new Value()); sinkRecordActiveCount.add(metricGroup.metricName(registry.sinkRecordActiveCountMax), new Max()); sinkRecordActiveCount.add(metricGroup.metricName(registry.sinkRecordActiveCountAvg), new Avg()); - partitionCount = metricGroup.metrics().sensor("partition-count"); + partitionCount = metricGroup.sensor("partition-count"); partitionCount.add(metricGroup.metricName(registry.sinkRecordPartitionCount), new Value()); - offsetSeqNum = metricGroup.metrics().sensor("offset-seq-number"); + offsetSeqNum = metricGroup.sensor("offset-seq-number"); offsetSeqNum.add(metricGroup.metricName(registry.sinkRecordOffsetCommitSeqNum), new Value()); - offsetCompletion = metricGroup.metrics().sensor("offset-commit-completion"); + offsetCompletion = metricGroup.sensor("offset-commit-completion"); offsetCompletion.add(metricGroup.metricName(registry.sinkRecordOffsetCommitCompletionRate), new Rate()); offsetCompletion.add(metricGroup.metricName(registry.sinkRecordOffsetCommitCompletionTotal), new Total()); - offsetCompletionSkip = metricGroup.metrics().sensor("offset-commit-completion-skip"); + offsetCompletionSkip = metricGroup.sensor("offset-commit-completion-skip"); offsetCompletionSkip.add(metricGroup.metricName(registry.sinkRecordOffsetCommitSkipRate), new Rate()); offsetCompletionSkip.add(metricGroup.metricName(registry.sinkRecordOffsetCommitSkipTotal), new Total()); - putBatchTime = metricGroup.metrics().sensor("put-batch-time"); + putBatchTime = metricGroup.sensor("put-batch-time"); putBatchTime.add(metricGroup.metricName(registry.sinkRecordPutBatchTimeMax), new Max()); putBatchTime.add(metricGroup.metricName(registry.sinkRecordPutBatchTimeAvg), new Avg()); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java index 6288767b994b6..6ef5ae3201cb0 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java @@ -531,7 +531,7 @@ public SourceTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeMax), new Max()); pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeAvg), new Avg()); - sourceRecordActiveCount = metricGroup.metrics().sensor("sink-record-active-count"); + sourceRecordActiveCount = metricGroup.sensor("source-record-active-count"); sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCount), new Value()); sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountMax), new Max()); sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountAvg), new Avg()); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java index 4ff4248b3a2f5..9568e787cc592 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java @@ -22,6 +22,7 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.common.MetricName; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.record.RecordBatch; @@ -32,6 +33,7 @@ import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.runtime.WorkerSinkTask.SinkTaskMetricsGroup; import org.apache.kafka.connect.sink.SinkConnector; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTask; @@ -75,6 +77,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -108,6 +111,7 @@ public class WorkerSinkTaskTest { private static final TaskConfig TASK_CONFIG = new TaskConfig(TASK_PROPS); private ConnectorTaskId taskId = new ConnectorTaskId("job", 0); + private ConnectorTaskId taskId1 = new ConnectorTaskId("job", 1); private TargetState initialState = TargetState.STARTED; private MockTime time; private WorkerSinkTask workerTask; @@ -1183,6 +1187,67 @@ public void testTopicsRegex() throws Exception { PowerMock.verifyAll(); } + @Test + public void testMetricsGroup() { + SinkTaskMetricsGroup group = new SinkTaskMetricsGroup(taskId, metrics); + SinkTaskMetricsGroup group1 = new SinkTaskMetricsGroup(taskId1, metrics); + for (int i = 0; i != 10; ++i) { + group.recordRead(1); + group.recordSend(2); + group.recordPut(3); + group.recordPartitionCount(4); + group.recordOffsetSequenceNumber(5); + } + Map committedOffsets = new HashMap<>(); + committedOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET + 1)); + group.recordCommittedOffsets(committedOffsets); + Map consumedOffsets = new HashMap<>(); + consumedOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET + 10)); + group.recordConsumedOffsets(consumedOffsets); + + for (int i = 0; i != 20; ++i) { + group1.recordRead(1); + group1.recordSend(2); + group1.recordPut(30); + group1.recordPartitionCount(40); + group1.recordOffsetSequenceNumber(50); + } + committedOffsets = new HashMap<>(); + committedOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET + 2)); + committedOffsets.put(TOPIC_PARTITION3, new OffsetAndMetadata(FIRST_OFFSET + 3)); + group1.recordCommittedOffsets(committedOffsets); + consumedOffsets = new HashMap<>(); + consumedOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET + 20)); + consumedOffsets.put(TOPIC_PARTITION3, new OffsetAndMetadata(FIRST_OFFSET + 30)); + group1.recordConsumedOffsets(consumedOffsets); + + assertEquals(0.333, metrics.currentMetricValueAsDouble(group.metricGroup(), "sink-record-read-rate"), 0.001d); + assertEquals(0.667, metrics.currentMetricValueAsDouble(group.metricGroup(), "sink-record-send-rate"), 0.001d); + assertEquals(9, metrics.currentMetricValueAsDouble(group.metricGroup(), "sink-record-active-count"), 0.001d); + assertEquals(4, metrics.currentMetricValueAsDouble(group.metricGroup(), "partition-count"), 0.001d); + assertEquals(5, metrics.currentMetricValueAsDouble(group.metricGroup(), "offset-commit-seq-no"), 0.001d); + assertEquals(3, metrics.currentMetricValueAsDouble(group.metricGroup(), "put-batch-max-time-ms"), 0.001d); + + // Close the group + group.close(); + + for (MetricName metricName : group.metricGroup().metrics().metrics().keySet()) { + // Metrics for this group should no longer exist + assertFalse(group.metricGroup().groupId().includes(metricName)); + } + // Sensors for this group should no longer exist + assertNull(group.metricGroup().metrics().getSensor("source-record-poll")); + assertNull(group.metricGroup().metrics().getSensor("source-record-write")); + assertNull(group.metricGroup().metrics().getSensor("poll-batch-time")); + + assertEquals(0.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "sink-record-read-rate"), 0.001d); + assertEquals(1.333, metrics.currentMetricValueAsDouble(group1.metricGroup(), "sink-record-send-rate"), 0.001d); + assertEquals(45, metrics.currentMetricValueAsDouble(group1.metricGroup(), "sink-record-active-count"), 0.001d); + assertEquals(40, metrics.currentMetricValueAsDouble(group1.metricGroup(), "partition-count"), 0.001d); + assertEquals(50, metrics.currentMetricValueAsDouble(group1.metricGroup(), "offset-commit-seq-no"), 0.001d); + assertEquals(30, metrics.currentMetricValueAsDouble(group1.metricGroup(), "put-batch-max-time-ms"), 0.001d); + } + private void expectInitializeTask() throws Exception { PowerMock.expectPrivate(workerTask, "createConsumer").andReturn(consumer); consumer.subscribe(EasyMock.eq(asList(TOPIC)), EasyMock.capture(rebalanceListener)); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java index 266c6add11008..25c2cb103529b 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.MetricName; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.record.InvalidRecordException; import org.apache.kafka.common.utils.Time; @@ -66,6 +67,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -88,6 +90,7 @@ public class WorkerSourceTaskTest extends ThreadedTest { private ExecutorService executor = Executors.newSingleThreadExecutor(); private ConnectorTaskId taskId = new ConnectorTaskId("job", 0); + private ConnectorTaskId taskId1 = new ConnectorTaskId("job", 1); private WorkerConfig config; private Plugins plugins; private MockConnectMetrics metrics; @@ -613,16 +616,47 @@ public Object answer() throws Throwable { @Test public void testMetricsGroup() { SourceTaskMetricsGroup group = new SourceTaskMetricsGroup(taskId, metrics); + SourceTaskMetricsGroup group1 = new SourceTaskMetricsGroup(taskId1, metrics); for (int i = 0; i != 10; ++i) { group.recordPoll(100, 1000 + i * 100); group.recordWrite(10); } + for (int i = 0; i != 20; ++i) { + group1.recordPoll(100, 1000 + i * 100); + group1.recordWrite(10); + } assertEquals(1900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-max-time-ms"), 0.001d); assertEquals(1450.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-avg-time-ms"), 0.001d); assertEquals(33.333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-rate"), 0.001d); assertEquals(1000, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-total"), 0.001d); assertEquals(3.3333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-rate"), 0.001d); assertEquals(100, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-total"), 0.001d); + assertEquals(900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-active-count"), 0.001d); + + // Close the group + group.close(); + + for (MetricName metricName : group.metricGroup().metrics().metrics().keySet()) { + // Metrics for this group should no longer exist + assertFalse(group.metricGroup().groupId().includes(metricName)); + } + // Sensors for this group should no longer exist + assertNull(group.metricGroup().metrics().getSensor("sink-record-read")); + assertNull(group.metricGroup().metrics().getSensor("sink-record-send")); + assertNull(group.metricGroup().metrics().getSensor("sink-record-active-count")); + assertNull(group.metricGroup().metrics().getSensor("partition-count")); + assertNull(group.metricGroup().metrics().getSensor("offset-seq-number")); + assertNull(group.metricGroup().metrics().getSensor("offset-commit-completion")); + assertNull(group.metricGroup().metrics().getSensor("offset-commit-completion-skip")); + assertNull(group.metricGroup().metrics().getSensor("put-batch-time")); + + assertEquals(2900.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-max-time-ms"), 0.001d); + assertEquals(1950.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-avg-time-ms"), 0.001d); + assertEquals(66.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-rate"), 0.001d); + assertEquals(2000, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-total"), 0.001d); + assertEquals(6.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-rate"), 0.001d); + assertEquals(200, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-total"), 0.001d); + assertEquals(1800.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-active-count"), 0.001d); } private CountDownLatch expectPolls(int minimum, final AtomicInteger count) throws InterruptedException { From 6541ab199395bfb06a7a020f7ab76d70855006c2 Mon Sep 17 00:00:00 2001 From: Ewen Cheslack-Postava Date: Tue, 6 Feb 2018 09:38:35 -0800 Subject: [PATCH 0010/1847] MINOR: Update release script with new remote, better error handling, correct mvn deploy profile Author: Ewen Cheslack-Postava Reviewers: Damian Guy , Ismael Juma Closes #4528 from ewencp/update-release-script --- release.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/release.py b/release.py index a21ffdc47d2a0..b008794e733d8 100755 --- a/release.py +++ b/release.py @@ -45,8 +45,8 @@ SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) # Location of the local git repository REPO_HOME = os.environ.get("%s_HOME" % CAPITALIZED_PROJECT_NAME, SCRIPT_DIR) -# Remote name which points to Apache git -PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache") +# Remote name, which points to Github by default +PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache-github") PREFS_FILE = os.path.join(SCRIPT_DIR, '.release-settings.json') delete_gitrefs = False @@ -77,6 +77,7 @@ def print_output(output): def cmd(action, cmd, *args, **kwargs): if isinstance(cmd, basestring) and not kwargs.get("shell", False): cmd = cmd.split() + allow_failure = kwargs.pop("allow_failure", False) stdin_log = "" if "stdin" in kwargs and isinstance(kwargs["stdin"], basestring): @@ -93,6 +94,9 @@ def cmd(action, cmd, *args, **kwargs): except subprocess.CalledProcessError as e: print_output(e.output) + if allow_failure: + return + print("*************************************************") print("*** First command failure occurred here. ***") print("*** Will now try to clean up working state. ***") @@ -128,7 +132,7 @@ def sftp_mkdir(dir): cd %s mkdir %s """ % (basedir, dirname) - cmd("Creating '%s' in '%s' in your Apache home directory if it does not exist (errors are ok if the directory already exists)" % (dirname, basedir), "sftp -b - %s@home.apache.org" % apache_id, stdin=cmd_str) + cmd("Creating '%s' in '%s' in your Apache home directory if it does not exist (errors are ok if the directory already exists)" % (dirname, basedir), "sftp -b - %s@home.apache.org" % apache_id, stdin=cmd_str, allow_failure=True) except subprocess.CalledProcessError: # This is ok. The command fails if the directory already exists pass @@ -389,7 +393,7 @@ def select_gpg_key(): fail("Retry again later") cmd("Building and uploading archives", "./gradlew uploadArchivesAll", cwd=kafka_dir, env=jdk7_env) cmd("Building and uploading archives", "./gradlew uploadCoreArchives_2_12 -PscalaVersion=2.12", cwd=kafka_dir, env=jdk8_env) -cmd("Building and uploading archives", "mvn deploy", cwd=streams_quickstart_dir, env=jdk7_env) +cmd("Building and uploading archives", "mvn deploy -Pgpg-signing", cwd=streams_quickstart_dir, env=jdk7_env) release_notification_props = { 'release_version': release_version, 'rc': rc, From 7c97b239a5261cf030984f71427287196e657d95 Mon Sep 17 00:00:00 2001 From: huxihx Date: Tue, 6 Feb 2018 10:40:51 -0800 Subject: [PATCH 0011/1847] KAFKA-6184; report a metric of the lag between the consumer offset ... Add `records-lead` and partition-level `{topic}-{partition}.records-lead-min|avg` for fetcher metrics. junrao Please kindly review. Thanks. Author: huxihx Reviewers: Jun Rao Closes #4191 from huxihx/KAFKA-6184 --- .../clients/consumer/internals/Fetcher.java | 47 +++++++- .../internals/FetcherMetricsRegistry.java | 19 ++- .../consumer/internals/SubscriptionState.java | 11 ++ .../consumer/internals/FetcherTest.java | 54 +++++++++ .../kafka/api/PlaintextConsumerTest.scala | 111 ++++++++++++++++++ 5 files changed, 237 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 32782ee53db70..0711d15cdb8cb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -45,6 +45,7 @@ import org.apache.kafka.common.metrics.stats.Count; import org.apache.kafka.common.metrics.stats.Max; import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.Min; import org.apache.kafka.common.metrics.stats.Value; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.BufferSupplier; @@ -587,6 +588,11 @@ private List> fetchRecords(PartitionRecords partitionRecord if (partitionLag != null) this.sensors.recordPartitionLag(partitionRecords.partition, partitionLag); + Long lead = subscriptions.partitionLead(partitionRecords.partition); + if (lead != null) { + this.sensors.recordPartitionLead(partitionRecords.partition, lead); + } + return partRecords; } else { // these records aren't next in line based on the last consumed position, ignore them @@ -871,6 +877,11 @@ private PartitionRecords parseCompletedFetch(CompletedFetch completedFetch) { subscriptions.updateHighWatermark(tp, partition.highWatermark); } + if (partition.logStartOffset >= 0) { + log.trace("Updating log start offset for partition {} to {}", tp, partition.logStartOffset); + subscriptions.updateLogStartOffset(tp, partition.logStartOffset); + } + if (partition.lastStableOffset >= 0) { log.trace("Updating last stable offset for partition {} to {}", tp, partition.lastStableOffset); subscriptions.updateLastStableOffset(tp, partition.lastStableOffset); @@ -945,7 +956,7 @@ private ConsumerRecord parseRecord(TopicPartition partition, @Override public void onAssignment(Set assignment) { - sensors.updatePartitionLagSensors(assignment); + sensors.updatePartitionLagAndLeadSensors(assignment); } public static Sensor throttleTimeSensor(Metrics metrics, FetcherMetricsRegistry metricsRegistry) { @@ -1261,6 +1272,7 @@ private static class FetchManagerMetrics { private final Sensor recordsFetched; private final Sensor fetchLatency; private final Sensor recordsFetchLag; + private final Sensor recordsFetchLead; private Set assignedPartitions; @@ -1287,6 +1299,9 @@ private FetchManagerMetrics(Metrics metrics, FetcherMetricsRegistry metricsRegis this.recordsFetchLag = metrics.sensor("records-lag"); this.recordsFetchLag.add(metrics.metricInstance(metricsRegistry.recordsLagMax), new Max()); + + this.recordsFetchLead = metrics.sensor("records-lead"); + this.recordsFetchLead.add(metrics.metricInstance(metricsRegistry.recordsLeadMin), new Min()); } private void recordTopicFetchMetrics(String topic, int bytes, int records) { @@ -1322,16 +1337,37 @@ private void recordTopicFetchMetrics(String topic, int bytes, int records) { recordsFetched.record(records); } - private void updatePartitionLagSensors(Set assignedPartitions) { + private void updatePartitionLagAndLeadSensors(Set assignedPartitions) { if (this.assignedPartitions != null) { for (TopicPartition tp : this.assignedPartitions) { - if (!assignedPartitions.contains(tp)) + if (!assignedPartitions.contains(tp)) { metrics.removeSensor(partitionLagMetricName(tp)); + metrics.removeSensor(partitionLeadMetricName(tp)); + } } } this.assignedPartitions = assignedPartitions; } + private void recordPartitionLead(TopicPartition tp, long lead) { + this.recordsFetchLead.record(lead); + + String name = partitionLeadMetricName(tp); + Sensor recordsLead = this.metrics.getSensor(name); + if (recordsLead == null) { + Map metricTags = new HashMap<>(2); + metricTags.put("topic", tp.topic().replace('.', '_')); + metricTags.put("partition", String.valueOf(tp.partition())); + + recordsLead = this.metrics.sensor(name); + + recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLead, metricTags), new Value()); + recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadMin, metricTags), new Min()); + recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadAvg, metricTags), new Avg()); + } + recordsLead.record(lead); + } + private void recordPartitionLag(TopicPartition tp, long lag) { this.recordsFetchLag.record(lag); @@ -1364,6 +1400,11 @@ private void recordPartitionLag(TopicPartition tp, long lag) { private static String partitionLagMetricName(TopicPartition tp) { return tp + ".records-lag"; } + + private static String partitionLeadMetricName(TopicPartition tp) { + return tp + ".records-lead"; + } + } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java index 301363a638b1f..6eb4fa20ff48f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetcherMetricsRegistry.java @@ -38,6 +38,7 @@ public class FetcherMetricsRegistry { public MetricNameTemplate fetchRequestRate; public MetricNameTemplate fetchRequestTotal; public MetricNameTemplate recordsLagMax; + public MetricNameTemplate recordsLeadMin; public MetricNameTemplate fetchThrottleTimeAvg; public MetricNameTemplate fetchThrottleTimeMax; public MetricNameTemplate topicFetchSizeAvg; @@ -50,6 +51,9 @@ public class FetcherMetricsRegistry { public MetricNameTemplate partitionRecordsLag; public MetricNameTemplate partitionRecordsLagMax; public MetricNameTemplate partitionRecordsLagAvg; + public MetricNameTemplate partitionRecordsLead; + public MetricNameTemplate partitionRecordsLeadMin; + public MetricNameTemplate partitionRecordsLeadAvg; // To remove in 2.0 public MetricNameTemplate partitionRecordsLagDeprecated; public MetricNameTemplate partitionRecordsLagMaxDeprecated; @@ -96,6 +100,8 @@ public FetcherMetricsRegistry(Set tags, String metricGrpPrefix) { this.recordsLagMax = new MetricNameTemplate("records-lag-max", groupName, "The maximum lag in terms of number of records for any partition in this window", tags); + this.recordsLeadMin = new MetricNameTemplate("records-lead-min", groupName, + "The minimum lead in terms of number of records for any partition in this window", tags); this.fetchThrottleTimeAvg = new MetricNameTemplate("fetch-throttle-time-avg", groupName, "The average throttle time in ms", tags); @@ -138,7 +144,12 @@ public FetcherMetricsRegistry(Set tags, String metricGrpPrefix) { "The max lag of the partition", partitionTags); this.partitionRecordsLagAvg = new MetricNameTemplate("records-lag-avg", groupName, "The average lag of the partition", partitionTags); - + this.partitionRecordsLead = new MetricNameTemplate("records-lead", groupName, + "The latest lead of the partition", partitionTags); + this.partitionRecordsLeadMin = new MetricNameTemplate("records-lead-min", groupName, + "The min lead of the partition", partitionTags); + this.partitionRecordsLeadAvg = new MetricNameTemplate("records-lead-avg", groupName, + "The average lead of the partition", partitionTags); } @@ -156,6 +167,7 @@ public List getAllTemplates() { fetchRequestRate, fetchRequestTotal, recordsLagMax, + recordsLeadMin, fetchThrottleTimeAvg, fetchThrottleTimeMax, topicFetchSizeAvg, @@ -170,7 +182,10 @@ public List getAllTemplates() { partitionRecordsLagMaxDeprecated, partitionRecordsLag, partitionRecordsLagAvg, - partitionRecordsLagMax + partitionRecordsLagMax, + partitionRecordsLead, + partitionRecordsLeadMin, + partitionRecordsLeadAvg ); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java index adced58d44ff5..822870713f4b3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java @@ -327,10 +327,19 @@ public Long partitionLag(TopicPartition tp, IsolationLevel isolationLevel) { return topicPartitionState.highWatermark == null ? null : topicPartitionState.highWatermark - topicPartitionState.position; } + public Long partitionLead(TopicPartition tp) { + TopicPartitionState topicPartitionState = assignedState(tp); + return topicPartitionState.logStartOffset == null ? null : topicPartitionState.position - topicPartitionState.logStartOffset; + } + public void updateHighWatermark(TopicPartition tp, long highWatermark) { assignedState(tp).highWatermark = highWatermark; } + public void updateLogStartOffset(TopicPartition tp, long logStartOffset) { + assignedState(tp).logStartOffset = logStartOffset; + } + public void updateLastStableOffset(TopicPartition tp, long lastStableOffset) { assignedState(tp).lastStableOffset = lastStableOffset; } @@ -435,6 +444,7 @@ private static Map partitionToStateMap(Coll private static class TopicPartitionState { private Long position; // last consumed position private Long highWatermark; // the high watermark from last fetch + private Long logStartOffset; // the log start offset private Long lastStableOffset; private OffsetAndMetadata committed; // last committed position private boolean paused; // whether this partition has been paused by the user @@ -444,6 +454,7 @@ public TopicPartitionState() { this.paused = false; this.position = null; this.highWatermark = null; + this.logStartOffset = null; this.lastStableOffset = null; this.committed = null; this.resetStrategy = null; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index a0205e7f19ec2..e81e3d1fc3ad1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1210,6 +1210,45 @@ public void testFetcherMetrics() { assertFalse(allMetrics.containsKey(partitionLagMetric)); } + @Test + public void testFetcherLeadMetric() { + subscriptions.assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName minLeadMetric = metrics.metricInstance(metricsRegistry.recordsLeadMin); + Map tags = new HashMap<>(2); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLeadMetric = metrics.metricName("records-lead", metricGroup, "", tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLeadMin = allMetrics.get(minLeadMetric); + + // recordsFetchLeadMin should be initialized to MAX_VALUE + assertEquals(Double.MAX_VALUE, recordsFetchLeadMin.value(), EPSILON); + + // recordsFetchLeadMin should be position - logStartOffset after receiving an empty FetchResponse + fetchRecords(tp0, MemoryRecords.EMPTY, Errors.NONE, 100L, -1L, 0L, 0); + assertEquals(0L, recordsFetchLeadMin.value(), EPSILON); + + KafkaMetric partitionLead = allMetrics.get(partitionLeadMetric); + assertEquals(0L, partitionLead.value(), EPSILON); + + // recordsFetchLeadMin should be position - logStartOffset after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) { + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + } + fetchRecords(tp0, builder.build(), Errors.NONE, 200L, -1L, 0L, 0); + assertEquals(0L, recordsFetchLeadMin.value(), EPSILON); + assertEquals(3L, partitionLead.value(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + assertFalse(allMetrics.containsKey(partitionLeadMetric)); + } + @Test public void testReadCommittedLagMetric() { Metrics metrics = new Metrics(); @@ -1430,6 +1469,14 @@ private Map>> fetchRecords( return fetcher.fetchedRecords(); } + private Map>> fetchRecords( + TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) { + assertEquals(1, fetcher.sendFetches()); + client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime)); + consumerClient.poll(0); + return fetcher.fetchedRecords(); + } + @Test public void testGetOffsetsForTimesTimeout() { try { @@ -2132,6 +2179,13 @@ private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records return new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); } + private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, long logStartOffset, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponse.PartitionData(error, hw, lastStableOffset, logStartOffset, null, records)); + return new FetchResponse(new LinkedHashMap<>(partitions), throttleTime); + } + private MetadataResponse newMetadataResponse(String topic, Errors error) { List partitionsMetadata = new ArrayList<>(); if (error == Errors.NONE) { diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala index 103a28ae67bba..a06e9e36528f3 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala @@ -1375,6 +1375,54 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(500, consumer0.committed(tp2).offset) } + @Test + def testPerPartitionLeadMetricsCleanUpWithSubscribe() { + val numMessages = 1000 + val topic2 = "topic2" + createTopic(topic2, 2, serverCount) + // send some messages. + sendRecords(numMessages, tp) + // Test subscribe + // Create a consumer and consumer some messages. + consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "testPerPartitionLeadMetricsCleanUpWithSubscribe") + consumerConfig.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "testPerPartitionLeadMetricsCleanUpWithSubscribe") + val consumer = new KafkaConsumer(this.consumerConfig, new ByteArrayDeserializer(), new ByteArrayDeserializer()) + try { + val listener0 = new TestConsumerReassignmentListener + consumer.subscribe(List(topic, topic2).asJava, listener0) + var records: ConsumerRecords[Array[Byte], Array[Byte]] = ConsumerRecords.empty() + TestUtils.waitUntilTrue(() => { + records = consumer.poll(100) + !records.records(tp).isEmpty + }, "Consumer did not consume any message before timeout.") + assertEquals("should be assigned once", 1, listener0.callsToAssigned) + // Verify the metric exist. + val tags1 = new util.HashMap[String, String]() + tags1.put("client-id", "testPerPartitionLeadMetricsCleanUpWithSubscribe") + tags1.put("topic", tp.topic()) + tags1.put("partition", String.valueOf(tp.partition())) + + val tags2 = new util.HashMap[String, String]() + tags2.put("client-id", "testPerPartitionLeadMetricsCleanUpWithSubscribe") + tags2.put("topic", tp2.topic()) + tags2.put("partition", String.valueOf(tp2.partition())) + val fetchLead0 = consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags1)) + assertNotNull(fetchLead0) + assertTrue(s"The lead should be ${records.count}", fetchLead0.metricValue() == records.count) + + // Remove topic from subscription + consumer.subscribe(List(topic2).asJava, listener0) + TestUtils.waitUntilTrue(() => { + consumer.poll(100) + listener0.callsToAssigned >= 2 + }, "Expected rebalance did not occur.") + // Verify the metric has gone + assertNull(consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags1))) + assertNull(consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags2))) + } finally { + consumer.close() + } + } @Test def testPerPartitionLagMetricsCleanUpWithSubscribe() { @@ -1426,6 +1474,41 @@ class PlaintextConsumerTest extends BaseConsumerTest { } } + @Test + def testPerPartitionLeadMetricsCleanUpWithAssign() { + val numMessages = 1000 + // Test assign + // send some messages. + sendRecords(numMessages, tp) + sendRecords(numMessages, tp2) + consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "testPerPartitionLeadMetricsCleanUpWithAssign") + consumerConfig.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "testPerPartitionLeadMetricsCleanUpWithAssign") + val consumer = new KafkaConsumer(this.consumerConfig, new ByteArrayDeserializer(), new ByteArrayDeserializer()) + try { + consumer.assign(List(tp).asJava) + var records: ConsumerRecords[Array[Byte], Array[Byte]] = ConsumerRecords.empty() + TestUtils.waitUntilTrue(() => { + records = consumer.poll(100) + !records.records(tp).isEmpty + }, "Consumer did not consume any message before timeout.") + // Verify the metric exist. + val tags = new util.HashMap[String, String]() + tags.put("client-id", "testPerPartitionLeadMetricsCleanUpWithAssign") + tags.put("topic", tp.topic()) + tags.put("partition", String.valueOf(tp.partition())) + val fetchLead = consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags)) + assertNotNull(fetchLead) + + assertTrue(s"The lead should be ${records.count}", records.count == fetchLead.metricValue()) + + consumer.assign(List(tp2).asJava) + TestUtils.waitUntilTrue(() => !consumer.poll(100).isEmpty, "Consumer did not consume any message before timeout.") + assertNull(consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags))) + } finally { + consumer.close() + } + } + @Test def testPerPartitionLagMetricsCleanUpWithAssign() { val numMessages = 1000 @@ -1467,6 +1550,34 @@ class PlaintextConsumerTest extends BaseConsumerTest { } } + @Test + def testPerPartitionLeadWithMaxPollRecords() { + val numMessages = 1000 + val maxPollRecords = 10 + sendRecords(numMessages, tp) + consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "testPerPartitionLeadWithMaxPollRecords") + consumerConfig.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "testPerPartitionLeadWithMaxPollRecords") + consumerConfig.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords.toString) + val consumer = new KafkaConsumer(this.consumerConfig, new ByteArrayDeserializer(), new ByteArrayDeserializer()) + consumer.assign(List(tp).asJava) + try { + var records: ConsumerRecords[Array[Byte], Array[Byte]] = ConsumerRecords.empty() + TestUtils.waitUntilTrue(() => { + records = consumer.poll(100) + !records.isEmpty + }, "Consumer did not consume any message before timeout.") + + val tags = new util.HashMap[String, String]() + tags.put("client-id", "testPerPartitionLeadWithMaxPollRecords") + tags.put("topic", tp.topic()) + tags.put("partition", String.valueOf(tp.partition())) + val lead = consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags)) + assertTrue(s"The lead should be $maxPollRecords", lead.metricValue() == maxPollRecords) + } finally { + consumer.close() + } + } + @Test def testPerPartitionLagWithMaxPollRecords() { val numMessages = 1000 From c32860b22427ca32a9e83e4f741b85062bb22140 Mon Sep 17 00:00:00 2001 From: Wladimir Schmidt Date: Tue, 6 Feb 2018 20:33:49 +0100 Subject: [PATCH 0012/1847] MINOR: exchange redundant Collections.addAll with parameterized constructor (#4521) * Exchange manual copy to collection with Collections.addAll call * Exchange redundant Collections.addAll with parameterized constructor call Reviewers: Guozhang Wang --- .../clients/consumer/internals/ConsumerMetrics.java | 3 +-- .../apache/kafka/clients/producer/ProducerConfig.java | 3 +-- .../clients/producer/internals/ProducerMetrics.java | 3 +-- .../org/apache/kafka/common/MetricNameTemplate.java | 6 +++--- .../apache/kafka/connect/runtime/AbstractHerder.java | 10 +++------- .../kafka/connect/runtime/WorkerSinkTaskContext.java | 4 ++-- .../kafka/connect/storage/KafkaConfigBackingStore.java | 6 ++---- .../kafka/connect/storage/KafkaOffsetBackingStore.java | 6 ++---- .../kafka/connect/storage/KafkaStatusBackingStore.java | 6 ++---- .../runtime/rest/resources/ConnectorsResourceTest.java | 3 +-- 10 files changed, 18 insertions(+), 32 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetrics.java index 3492323170841..e58db82ee3cd6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerMetrics.java @@ -37,8 +37,7 @@ public ConsumerMetrics(String metricGroupPrefix) { } private List getAllTemplates() { - List l = new ArrayList<>(); - l.addAll(this.fetcherMetrics.getAllTemplates()); + List l = new ArrayList<>(this.fetcherMetrics.getAllTemplates()); return l; } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index 6428dc42d491b..aa3147b28b8db 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -340,8 +340,7 @@ protected Map postProcessParsedConfig(final Map public static Map addSerializerToConfig(Map configs, Serializer keySerializer, Serializer valueSerializer) { - Map newConfigs = new HashMap<>(); - newConfigs.putAll(configs); + Map newConfigs = new HashMap<>(configs); if (keySerializer != null) newConfigs.put(KEY_SERIALIZER_CLASS_CONFIG, keySerializer.getClass()); if (valueSerializer != null) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetrics.java index f78f3d6647ca1..030a23299b93e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerMetrics.java @@ -36,8 +36,7 @@ public ProducerMetrics(Metrics metrics) { } private List getAllTemplates() { - List l = new ArrayList<>(); - l.addAll(this.senderMetrics.allTemplates()); + List l = new ArrayList<>(this.senderMetrics.allTemplates()); return l; } diff --git a/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java b/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java index 1b1de71037d1f..abe121f7ac972 100644 --- a/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java +++ b/clients/src/main/java/org/apache/kafka/common/MetricNameTemplate.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.common; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; @@ -65,9 +66,8 @@ public MetricNameTemplate(String name, String group, String description, String. private static LinkedHashSet getTags(String... keys) { LinkedHashSet tags = new LinkedHashSet<>(); - - for (int i = 0; i < keys.length; i++) - tags.add(keys[i]); + + Collections.addAll(tags, keys); return tags; } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java index b913f9ed65e25..424b4749fe2a5 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java @@ -249,10 +249,6 @@ public ConfigInfos validateConnectorConfig(Map connectorProps) { if (connType == null) throw new BadRequestException("Connector config " + connectorProps + " contains no connector type"); - List configValues = new ArrayList<>(); - Map configKeys = new LinkedHashMap<>(); - Set allGroups = new LinkedHashSet<>(); - Connector connector = getConnector(connType); ClassLoader savedLoader = plugins().compareAndSwapLoaders(connector); try { @@ -269,9 +265,9 @@ public ConfigInfos validateConnectorConfig(Map connectorProps) { enrichedConfigDef, connectorProps ); - configValues.addAll(validatedConnectorConfig.values()); - configKeys.putAll(enrichedConfigDef.configKeys()); - allGroups.addAll(enrichedConfigDef.groups()); + List configValues = new ArrayList<>(validatedConnectorConfig.values()); + Map configKeys = new LinkedHashMap<>(enrichedConfigDef.configKeys()); + Set allGroups = new LinkedHashSet<>(enrichedConfigDef.groups()); // do custom connector-specific validation Config config = connector.validate(connectorProps); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java index 64c8fff52d670..80764037579df 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java @@ -22,6 +22,7 @@ import org.apache.kafka.connect.sink.SinkTaskContext; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -90,8 +91,7 @@ public void pause(TopicPartition... partitions) { throw new IllegalWorkerStateException("SinkTaskContext may not be used to pause consumption until the task is initialized"); } try { - for (TopicPartition partition : partitions) - pausedPartitions.add(partition); + Collections.addAll(pausedPartitions, partitions); consumer.pause(Arrays.asList(partitions)); } catch (IllegalStateException e) { throw new IllegalWorkerStateException("SinkTasks may not pause partitions that are not currently assigned to them.", e); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java index dd69c300b69b3..5c2c72c8d1b73 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java @@ -406,14 +406,12 @@ public void putTargetState(String connector, TargetState state) { // package private for testing KafkaBasedLog setupAndCreateKafkaBasedLog(String topic, final WorkerConfig config) { - Map producerProps = new HashMap<>(); - producerProps.putAll(config.originals()); + Map producerProps = new HashMap<>(config.originals()); producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE); - Map consumerProps = new HashMap<>(); - consumerProps.putAll(config.originals()); + Map consumerProps = new HashMap<>(config.originals()); consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java index cb4f0897887e2..2e4e5232dfd91 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java @@ -67,14 +67,12 @@ public void configure(final WorkerConfig config) { data = new HashMap<>(); - Map producerProps = new HashMap<>(); - producerProps.putAll(config.originals()); + Map producerProps = new HashMap<>(config.originals()); producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE); - Map consumerProps = new HashMap<>(); - consumerProps.putAll(config.originals()); + Map consumerProps = new HashMap<>(config.originals()); consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java index 11f951a73c348..ca744324740f5 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java @@ -124,14 +124,12 @@ public void configure(final WorkerConfig config) { if (topic.equals("")) throw new ConfigException("Must specify topic for connector status."); - Map producerProps = new HashMap<>(); - producerProps.putAll(config.originals()); + Map producerProps = new HashMap<>(config.originals()); producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); producerProps.put(ProducerConfig.RETRIES_CONFIG, 0); // we handle retries in this class - Map consumerProps = new HashMap<>(); - consumerProps.putAll(config.originals()); + Map consumerProps = new HashMap<>(config.originals()); consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java index cc4608064ed1e..f84cd258fd4ef 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java @@ -130,8 +130,7 @@ public void setUp() throws NoSuchMethodException { } private static final Map getConnectorConfig(Map mapToClone) { - Map result = new HashMap<>(); - result.putAll(mapToClone); + Map result = new HashMap<>(mapToClone); return result; } From a88db9595910a1a7f249f7c9f3effb14636955c7 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 6 Feb 2018 13:14:33 -0800 Subject: [PATCH 0013/1847] KAFKA-4750: Bypass null value and treat it as deletes (#4508) Here is the new rule for handling nulls: * in the interface store, put(key, null) are handled normally and value serde will still be applied to null, hence needs to handle null values * in the inner bytes store, null bytes after serialization will be treated as deletes. * in the interface store, if null bytes get returned in get(key), it indicate the key is not available; and hence serde will be avoided and null object will be returned. More changes: * Update javadocs, add unit tests accordingly; augment MockContext to set serdes for the newly added tests. * Fixed a discovered bug which is exposed by the newly added tests. * Use the new API to remove all old APIs in the existing state store tests. * Remove SerializedKeyValueIterator since it is not used any more. This is originally contributed by @evis. Reviewers: Bill Bejeck , Matthias J. Sax , Damian Guy --- .../kafka/streams/kstream/Materialized.java | 3 +- .../state/KeyValueBytesStoreSupplier.java | 5 + .../kafka/streams/state/KeyValueStore.java | 9 +- .../state/SessionBytesStoreSupplier.java | 7 +- .../kafka/streams/state/SessionStore.java | 9 +- .../apache/kafka/streams/state/Stores.java | 9 +- .../state/WindowBytesStoreSupplier.java | 7 +- .../kafka/streams/state/WindowStore.java | 5 +- .../state/internals/CachingKeyValueStore.java | 30 ++++-- .../state/internals/CachingSessionStore.java | 5 +- .../state/internals/CachingWindowStore.java | 3 +- .../ChangeLoggingKeyValueBytesStore.java | 11 +++ .../internals/InMemoryKeyValueStore.java | 22 +++-- .../internals/InnerMeteredKeyValueStore.java | 6 +- .../state/internals/MemoryLRUCache.java | 33 ++++--- .../internals/MemoryNavigableLRUCache.java | 7 -- .../internals/MeteredKeyValueBytesStore.java | 10 +- .../state/internals/MeteredSessionStore.java | 14 ++- .../streams/state/internals/RocksDBStore.java | 12 +-- .../state/internals/SegmentedBytesStore.java | 4 +- .../internals/SerializedKeyValueIterator.java | 70 -------------- .../internals/WindowStoreIteratorWrapper.java | 5 +- .../WrappedSessionStoreIterator.java | 3 +- .../kstream/internals/KTableFilterTest.java | 18 ++-- .../internals/AbstractKeyValueStoreTest.java | 75 +++++++++++++-- .../internals/CachingKeyValueStoreTest.java | 26 ++--- .../InMemoryKeyValueLoggedStoreTest.java | 23 +---- .../internals/InMemoryKeyValueStoreTest.java | 27 +++--- .../internals/InMemoryLRUCacheStoreTest.java | 29 +++--- .../internals/RocksDBKeyValueStoreTest.java | 31 ++---- .../SerializedKeyValueIteratorTest.java | 94 ------------------- .../kafka/test/MockProcessorContext.java | 12 ++- 32 files changed, 269 insertions(+), 355 deletions(-) delete mode 100644 streams/src/main/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIterator.java delete mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIteratorTest.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Materialized.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Materialized.java index 48dd12e08c313..89603fa7d3af1 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Materialized.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Materialized.java @@ -160,7 +160,8 @@ public static Materialized with(final Serd * Set the valueSerde the materialized {@link StateStore} will use. * * @param valueSerde the value {@link Serde} to use. If the {@link Serde} is null, then the default value - * serde from configs will be used + * serde from configs will be used. If the serialized bytes is null for put operations, + * it is treated as delete operation * @return itself */ public Materialized withValueSerde(final Serde valueSerde) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueBytesStoreSupplier.java index 73e6732de9734..b12e94ec382cc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueBytesStoreSupplier.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueBytesStoreSupplier.java @@ -20,6 +20,11 @@ /** * A store supplier that can be used to create one or more {@link KeyValueStore KeyValueStore} instances of type <Byte, byte[]>. + * + * For any stores implementing the {@link KeyValueStore KeyValueStore} interface, null value bytes are considered as "not exist". This means: + * + * 1. Null value bytes in put operations should be treated as delete. + * 2. If the key does not exist, get operations should return null value bytes. */ public interface KeyValueBytesStoreSupplier extends StoreSupplier> { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueStore.java index 129ae6b063498..3685229083778 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueStore.java @@ -33,7 +33,8 @@ public interface KeyValueStore extends StateStore, ReadOnlyKeyValueStore extends StateStore, ReadOnlyKeyValueStore extends StateStore, ReadOnlyKeyValueStore> entries); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/SessionBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/SessionBytesStoreSupplier.java index e5523da99d2b9..04b0ceb1ecfaa 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/SessionBytesStoreSupplier.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/SessionBytesStoreSupplier.java @@ -19,7 +19,12 @@ import org.apache.kafka.common.utils.Bytes; /** - * A store supplier that can be used to create one or more {@link SessionStore SessionStore>} instances of type <Byte, byte[]>. + * A store supplier that can be used to create one or more {@link SessionStore SessionStore} instances of type <Byte, byte[]>. + * + * For any stores implementing the {@link SessionStore SessionStore} interface, null value bytes are considered as "not exist". This means: + * + * 1. Null value bytes in put operations should be treated as delete. + * 2. Null value bytes should never be returned in range query results. */ public interface SessionBytesStoreSupplier extends StoreSupplier> { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java index fcbce5f849732..c1a699317ebee 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java @@ -33,7 +33,8 @@ public interface SessionStore extends StateStore, ReadOnlySessionStore extends StateStore, ReadOnlySessionStore extends StateStore, ReadOnlySessionStore sessionKey, final AGG aggregate); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java index 0ce6d9e31e276..0aaa2b23738ce 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java @@ -198,7 +198,8 @@ public static SessionBytesStoreSupplier persistentSessionStore(final String name * Creates a {@link StoreBuilder} that can be used to build a {@link WindowStore}. * @param supplier a {@link WindowBytesStoreSupplier} (cannot be {@code null}) * @param keySerde the key serde to use - * @param valueSerde the value serde to use + * @param valueSerde the value serde to use; if the serialized bytes is null for put operations, + * it is treated as delete * @param key type * @param value type * @return an instance of {@link StoreBuilder} than can build a {@link WindowStore} @@ -214,7 +215,8 @@ public static StoreBuilder> windowStoreBuilder(final Wi * Creates a {@link StoreBuilder} than can be used to build a {@link KeyValueStore}. * @param supplier a {@link KeyValueBytesStoreSupplier} (cannot be {@code null}) * @param keySerde the key serde to use - * @param valueSerde the value serde to use + * @param valueSerde the value serde to use; if the serialized bytes is null for put operations, + * it is treated as delete * @param key type * @param value type * @return an instance of a {@link StoreBuilder} that can build a {@link KeyValueStore} @@ -230,7 +232,8 @@ public static StoreBuilder> keyValueStoreBuilder(fina * Creates a {@link StoreBuilder} that can be used to build a {@link SessionStore}. * @param supplier a {@link SessionBytesStoreSupplier} (cannot be {@code null}) * @param keySerde the key serde to use - * @param valueSerde the value serde to use + * @param valueSerde the value serde to use; if the serialized bytes is null for put operations, + * it is treated as delete * @param key type * @param value type * @return an instance of {@link StoreBuilder} than can build a {@link SessionStore} diff --git a/streams/src/main/java/org/apache/kafka/streams/state/WindowBytesStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/WindowBytesStoreSupplier.java index 5fbe6b04d63cd..9cf70c2f89cf5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/WindowBytesStoreSupplier.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/WindowBytesStoreSupplier.java @@ -19,7 +19,12 @@ import org.apache.kafka.common.utils.Bytes; /** - * A store supplier that can be used to create one or more {@link WindowStore WindowStore>} instances of type <Byte, byte[]>. + * A store supplier that can be used to create one or more {@link WindowStore WindowStore} instances of type <Byte, byte[]>. + * + * For any stores implementing the {@link WindowStore WindowStore} interface, null value bytes are considered as "not exist". This means: + * + * 1. Null value bytes in put operations should be treated as delete. + * 2. Null value bytes should never be returned in range query results. */ public interface WindowBytesStoreSupplier extends StoreSupplier> { /** diff --git a/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java index 6a4d5f6599206..d2f6ad8c5b8f4 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/WindowStore.java @@ -30,7 +30,8 @@ public interface WindowStore extends StateStore, ReadOnlyWindowStore * Put a key-value pair with the current wall-clock time as the timestamp * into the corresponding window * @param key The key to associate the value to - * @param value The value, it can be null + * @param value The value to update, it can be null; + * if the serialized bytes are also null it is interpreted as deletes * @throws NullPointerException If null is used for key. */ void put(K key, V value); @@ -38,7 +39,7 @@ public interface WindowStore extends StateStore, ReadOnlyWindowStore /** * Put a key-value pair with the given timestamp into the corresponding window * @param key The key to associate the value to - * @param value The value, it can be null + * @param value The value; can be null * @throws NullPointerException If null is used for key. */ void put(K key, V value, long timestamp); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java index f9ab3f1244959..82a6ac785a8eb 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java @@ -91,7 +91,12 @@ private void putAndMaybeForward(final ThreadCache.DirtyEntry entry, final Intern try { context.setRecordContext(entry.recordContext()); if (flushListener != null) { - final V oldValue = sendOldValues ? serdes.valueFrom(underlying.get(entry.key())) : null; + V oldValue = null; + if (sendOldValues) { + final byte[] oldBytesValue = underlying.get(entry.key()); + oldValue = oldBytesValue == null ? null : serdes.valueFrom(oldBytesValue); + } + // we rely on underlying store to handle null new value bytes as deletes underlying.put(entry.key(), entry.newValue()); flushListener.apply(serdes.keyFrom(entry.key().get()), serdes.valueFrom(entry.newValue()), @@ -141,6 +146,7 @@ public boolean isOpen() { @Override public byte[] get(final Bytes key) { + Objects.requireNonNull(key, "key cannot be null"); validateStoreOpen(); Lock theLock; if (Thread.currentThread().equals(streamThread)) { @@ -150,7 +156,6 @@ public byte[] get(final Bytes key) { } theLock.lock(); try { - Objects.requireNonNull(key); return getInternal(key); } finally { theLock.unlock(); @@ -212,15 +217,15 @@ public void put(final Bytes key, final byte[] value) { validateStoreOpen(); lock.writeLock().lock(); try { + // for null bytes, we still put it into cache indicating tombstones putInternal(key, value); } finally { lock.writeLock().unlock(); } } - private void putInternal(final Bytes rawKey, final byte[] value) { - Objects.requireNonNull(rawKey, "key cannot be null"); - cache.put(cacheName, rawKey, new LRUCacheEntry(value, true, context.offset(), + private void putInternal(final Bytes key, final byte[] value) { + cache.put(cacheName, key, new LRUCacheEntry(value, true, context.offset(), context.timestamp(), context.partition(), context.topic())); } @@ -242,9 +247,11 @@ public byte[] putIfAbsent(final Bytes key, final byte[] value) { @Override public void putAll(final List> entries) { + validateStoreOpen(); lock.writeLock().lock(); try { for (KeyValue entry : entries) { + Objects.requireNonNull(entry.key, "key cannot be null"); put(entry.key, entry.value); } } finally { @@ -254,19 +261,22 @@ public void putAll(final List> entries) { @Override public byte[] delete(final Bytes key) { + Objects.requireNonNull(key, "key cannot be null"); validateStoreOpen(); lock.writeLock().lock(); try { - Objects.requireNonNull(key); - final byte[] v = getInternal(key); - cache.delete(cacheName, key); - underlying.delete(key); - return v; + return deleteInternal(key); } finally { lock.writeLock().unlock(); } } + private byte[] deleteInternal(final Bytes key) { + final byte[] v = getInternal(key); + putInternal(key, null); + return v; + } + KeyValueStore underlying() { return underlying; } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java index 05851e5a846ad..31b9d75ef8d35 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java @@ -160,8 +160,6 @@ public KeyValueIterator, byte[]> fetch(final Bytes from, final B return findSessions(from, to, 0, Long.MAX_VALUE); } - - private void putAndMaybeForward(final ThreadCache.DirtyEntry entry, final InternalProcessorContext context) { final Bytes binaryKey = cacheFunction.key(entry.key()); final RecordContext current = context.recordContext(); @@ -183,8 +181,7 @@ private void putAndMaybeForward(final ThreadCache.DirtyEntry entry, final Intern } private AGG fetchPrevious(final Bytes rawKey, final Window window) { - try (final KeyValueIterator, byte[]> iterator = bytesStore - .findSessions(rawKey, window.start(), window.end())) { + try (final KeyValueIterator, byte[]> iterator = bytesStore.findSessions(rawKey, window.start(), window.end())) { if (!iterator.hasNext()) { return null; } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java index 75acd77bc9273..ad0bd99ecb47b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java @@ -112,8 +112,7 @@ private void maybeForward(final ThreadCache.DirtyEntry entry, context.setRecordContext(entry.recordContext()); try { final V oldValue = sendOldValues ? fetchPrevious(key, windowedKey.window().start()) : null; - flushListener.apply(windowedKey, - serdes.valueFrom(entry.newValue()), oldValue); + flushListener.apply(windowedKey, serdes.valueFrom(entry.newValue()), oldValue); } finally { context.setRecordContext(current); } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStore.java index 94ee275a3bfb3..0fdd3e0e658ce 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStore.java @@ -45,6 +45,17 @@ public void init(final ProcessorContext context, final StateStore root) { ProcessorStateManager.storeChangelogTopic( context.applicationId(), inner.name()))); + + // if the inner store is an LRU cache, add the eviction listener to log removed record + if (inner instanceof MemoryLRUCache) { + ((MemoryLRUCache) inner).whenEldestRemoved(new MemoryLRUCache.EldestEntryRemovalListener() { + @Override + public void apply(Bytes key, byte[] value) { + // pass null to indicate removal + changeLogger.logChange(key, null); + } + }); + } } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java index 929d584fdefeb..2cdfc4b5f1e2c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java @@ -63,7 +63,7 @@ public String name() { @Override @SuppressWarnings("unchecked") - public void init(ProcessorContext context, StateStore root) { + public void init(final ProcessorContext context, final StateStore root) { // construct the serde this.serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), name), @@ -99,17 +99,21 @@ public boolean isOpen() { } @Override - public synchronized V get(K key) { + public synchronized V get(final K key) { return this.map.get(key); } @Override - public synchronized void put(K key, V value) { - this.map.put(key, value); + public synchronized void put(final K key, final V value) { + if (value == null) { + this.map.remove(key); + } else { + this.map.put(key, value); + } } @Override - public synchronized V putIfAbsent(K key, V value) { + public synchronized V putIfAbsent(final K key, final V value) { V originalValue = get(key); if (originalValue == null) { put(key, value); @@ -118,18 +122,18 @@ public synchronized V putIfAbsent(K key, V value) { } @Override - public synchronized void putAll(List> entries) { + public synchronized void putAll(final List> entries) { for (KeyValue entry : entries) put(entry.key, entry.value); } @Override - public synchronized V delete(K key) { + public synchronized V delete(final K key) { return this.map.remove(key); } @Override - public synchronized KeyValueIterator range(K from, K to) { + public synchronized KeyValueIterator range(final K from, final K to) { return new DelegatingPeekingKeyValueIterator<>(name, new InMemoryKeyValueIterator<>(this.map.subMap(from, true, to, true).entrySet().iterator())); } @@ -158,7 +162,7 @@ public void close() { private static class InMemoryKeyValueIterator implements KeyValueIterator { private final Iterator> iter; - private InMemoryKeyValueIterator(Iterator> iter) { + private InMemoryKeyValueIterator(final Iterator> iter) { this.iter = iter; } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InnerMeteredKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InnerMeteredKeyValueStore.java index d076a0526bcd9..40e2d431e9d44 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InnerMeteredKeyValueStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InnerMeteredKeyValueStore.java @@ -75,9 +75,9 @@ interface TypeConverter { // always wrap the store with the metered store InnerMeteredKeyValueStore(final KeyValueStore inner, - final String metricScope, - final TypeConverter typeConverter, - final Time time) { + final String metricScope, + final TypeConverter typeConverter, + final Time time) { super(inner); this.inner = inner; this.metricScope = metricScope; diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryLRUCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryLRUCache.java index f1aa96f2c1959..2785540ec5437 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryLRUCache.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryLRUCache.java @@ -61,9 +61,12 @@ public interface EldestEntryRemovalListener { // in the future we should augment the StateRestoreCallback with onComplete etc to better resolve this. private volatile boolean open = true; - EldestEntryRemovalListener listener; + private EldestEntryRemovalListener listener; - MemoryLRUCache(String name, final int maxCacheSize, Serde keySerde, Serde valueSerde) { + MemoryLRUCache(final String name, + final int maxCacheSize, + final Serde keySerde, + final Serde valueSerde) { this.name = name; this.keySerde = keySerde; this.valueSerde = valueSerde; @@ -87,7 +90,7 @@ KeyValueStore enableLogging() { return new InMemoryKeyValueLoggedStore<>(this, keySerde, valueSerde); } - MemoryLRUCache whenEldestRemoved(EldestEntryRemovalListener listener) { + MemoryLRUCache whenEldestRemoved(final EldestEntryRemovalListener listener) { this.listener = listener; return this; @@ -114,7 +117,7 @@ public void restore(byte[] key, byte[] value) { restoring = true; // check value for null, to avoid deserialization error. if (value == null) { - put(serdes.keyFrom(key), null); + delete(serdes.keyFrom(key)); } else { put(serdes.keyFrom(key), serdes.valueFrom(value)); } @@ -134,19 +137,24 @@ public boolean isOpen() { } @Override - public synchronized V get(K key) { + public synchronized V get(final K key) { Objects.requireNonNull(key); + return this.map.get(key); } @Override - public synchronized void put(K key, V value) { + public synchronized void put(final K key, final V value) { Objects.requireNonNull(key); - this.map.put(key, value); + if (value == null) { + this.map.remove(key); + } else { + this.map.put(key, value); + } } @Override - public synchronized V putIfAbsent(K key, V value) { + public synchronized V putIfAbsent(final K key, final V value) { Objects.requireNonNull(key); V originalValue = get(key); if (originalValue == null) { @@ -156,23 +164,22 @@ public synchronized V putIfAbsent(K key, V value) { } @Override - public void putAll(List> entries) { + public void putAll(final List> entries) { for (KeyValue entry : entries) put(entry.key, entry.value); } @Override - public synchronized V delete(K key) { + public synchronized V delete(final K key) { Objects.requireNonNull(key); - V value = this.map.remove(key); - return value; + return this.map.remove(key); } /** * @throws UnsupportedOperationException */ @Override - public KeyValueIterator range(K from, K to) { + public KeyValueIterator range(final K from, final K to) { throw new UnsupportedOperationException("MemoryLRUCache does not support range() function."); } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java index 268042d95d3f7..9500e8e246a91 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java @@ -31,13 +31,6 @@ public MemoryNavigableLRUCache(String name, final int maxCacheSize, Serde key super(name, maxCacheSize, keySerde, valueSerde); } - @Override - public MemoryNavigableLRUCache whenEldestRemoved(EldestEntryRemovalListener listener) { - this.listener = listener; - - return this; - } - @Override public KeyValueIterator range(K from, K to) { final TreeMap treeMap = toTreeMap(); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStore.java index 35647b730aa44..3a30c10c8a115 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStore.java @@ -62,9 +62,7 @@ public Bytes innerKey(final K key) { @Override public byte[] innerValue(final V value) { - if (value == null) { - return null; - } + // do not check on null, but rely on user serde to handle it return serdes.rawValue(value); } @@ -80,12 +78,12 @@ public List> innerEntries(final List> fro @Override public V outerValue(final byte[] value) { - return serdes.valueFrom(value); + return value == null ? null : serdes.valueFrom(value); } @Override - public KeyValue outerKeyValue(final KeyValue from) { - return KeyValue.pair(serdes.keyFrom(from.key.get()), serdes.valueFrom(from.value)); + public KeyValue outerKeyValue(final KeyValue keyValue) { + return KeyValue.pair(serdes.keyFrom(keyValue.key.get()), keyValue.value == null ? null : serdes.valueFrom(keyValue.value)); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java index 8d9065cb6ae1c..1a9ac20e3b3e4 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java @@ -92,7 +92,7 @@ public KeyValueIterator, V> findSessions(final K key, final long earliestSessionEndTime, final long latestSessionStartTime) { Objects.requireNonNull(key, "key cannot be null"); - final Bytes bytesKey = Bytes.wrap(serdes.rawKey(key)); + final Bytes bytesKey = keyBytes(key); return new MeteredWindowedKeyValueIterator<>(inner.findSessions(bytesKey, earliestSessionEndTime, latestSessionStartTime), @@ -109,8 +109,8 @@ public KeyValueIterator, V> findSessions(final K keyFrom, final long latestSessionStartTime) { Objects.requireNonNull(keyFrom, "keyFrom cannot be null"); Objects.requireNonNull(keyTo, "keyTo cannot be null"); - final Bytes bytesKeyFrom = Bytes.wrap(serdes.rawKey(keyFrom)); - final Bytes bytesKeyTo = Bytes.wrap(serdes.rawKey(keyTo)); + final Bytes bytesKeyFrom = keyBytes(keyFrom); + final Bytes bytesKeyTo = keyBytes(keyTo); return new MeteredWindowedKeyValueIterator<>(inner.findSessions(bytesKeyFrom, bytesKeyTo, earliestSessionEndTime, @@ -126,7 +126,7 @@ public void remove(final Windowed sessionKey) { Objects.requireNonNull(sessionKey, "sessionKey can't be null"); final long startNs = time.nanoseconds(); try { - final Bytes key = Bytes.wrap(serdes.rawKey(sessionKey.key())); + final Bytes key = keyBytes(sessionKey.key()); inner.remove(new Windowed<>(key, sessionKey.window())); } finally { this.metrics.recordLatency(removeTime, startNs, time.nanoseconds()); @@ -138,13 +138,17 @@ public void put(final Windowed sessionKey, final V aggregate) { Objects.requireNonNull(sessionKey, "sessionKey can't be null"); long startNs = time.nanoseconds(); try { - final Bytes key = Bytes.wrap(serdes.rawKey(sessionKey.key())); + final Bytes key = keyBytes(sessionKey.key()); this.inner.put(new Windowed<>(key, sessionKey.window()), serdes.rawValue(aggregate)); } finally { this.metrics.recordLatency(this.putTime, startNs, time.nanoseconds()); } } + private Bytes keyBytes(final K key) { + return Bytes.wrap(serdes.rawKey(key)); + } + @Override public KeyValueIterator, V> fetch(final K key) { Objects.requireNonNull(key, "key cannot be null"); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java index 4c7e61bb63f59..67ec9155805a5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java @@ -229,11 +229,7 @@ public boolean isOpen() { public synchronized V get(K key) { validateStoreOpen(); byte[] byteValue = getInternal(serdes.rawKey(key)); - if (byteValue == null) { - return null; - } else { - return serdes.valueFrom(byteValue); - } + return byteValue == null ? null : serdes.valueFrom(byteValue); } private void validateStoreOpen() { @@ -341,11 +337,11 @@ public void putAll(List> entries) { for (KeyValue entry : entries) { Objects.requireNonNull(entry.key, "key cannot be null"); final byte[] rawKey = serdes.rawKey(entry.key); - if (entry.value == null) { + final byte[] rawValue = serdes.rawValue(entry.value); + if (rawValue == null) { batch.remove(rawKey); } else { - final byte[] value = serdes.rawValue(entry.value); - batch.put(rawKey, value); + batch.put(rawKey, rawValue); } } db.write(wOptions, batch); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java index 19cf319bf2584..a53318528e652 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java @@ -62,8 +62,8 @@ public interface SegmentedBytesStore extends StateStore { /** * Gets all the key-value pairs that belong to the windows within in the given time range. * - * @param timeFrom the beginning of the time slot from which to search - * @param timeTo the end of the time slot from which to search + * @param from the beginning of the time slot from which to search + * @param to the end of the time slot from which to search * @return an iterator over windowed key-value pairs {@code , value>} * @throws InvalidStateStoreException if the store is not initialized * @throws NullPointerException if null is used for any key diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIterator.java deleted file mode 100644 index 2c050f5578aeb..0000000000000 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIterator.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.streams.state.internals; - -import org.apache.kafka.common.utils.Bytes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.state.KeyValueIterator; -import org.apache.kafka.streams.state.StateSerdes; - -import java.util.NoSuchElementException; - -class SerializedKeyValueIterator implements KeyValueIterator { - - private final KeyValueIterator bytesIterator; - private final StateSerdes serdes; - - SerializedKeyValueIterator(final KeyValueIterator bytesIterator, - final StateSerdes serdes) { - - this.bytesIterator = bytesIterator; - this.serdes = serdes; - } - - @Override - public void close() { - bytesIterator.close(); - } - - @Override - public K peekNextKey() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - final Bytes bytes = bytesIterator.peekNextKey(); - return serdes.keyFrom(bytes.get()); - } - - @Override - public boolean hasNext() { - return bytesIterator.hasNext(); - } - - @Override - public KeyValue next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - final KeyValue next = bytesIterator.next(); - return KeyValue.pair(serdes.keyFrom(next.key.get()), serdes.valueFrom(next.value)); - } - - @Override - public void remove() { - throw new UnsupportedOperationException("remove() is not supported in " + getClass().getName()); - } -} diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java index 4fd6f3eeef5d2..4cb85d6d7600d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreIteratorWrapper.java @@ -149,8 +149,9 @@ private static class WrappedKeyValueIterator implements KeyValueIterator serdes; final long windowSize; - WrappedKeyValueIterator( - KeyValueIterator bytesIterator, StateSerdes serdes, long windowSize) { + WrappedKeyValueIterator(final KeyValueIterator bytesIterator, + final StateSerdes serdes, + final long windowSize) { this.bytesIterator = bytesIterator; this.serdes = serdes; this.windowSize = windowSize; diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java index 6fd963644011f..c5ea70bac80ac 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappedSessionStoreIterator.java @@ -77,7 +77,8 @@ public boolean hasNext() { @Override public KeyValue, V> next() { final KeyValue next = bytesIterator.next(); - return KeyValue.pair(SessionKeySerde.from(next.key.get(), serdes.keyDeserializer(), serdes.topic()), serdes.valueFrom(next.value)); + return KeyValue.pair(SessionKeySerde.from(next.key.get(), serdes.keyDeserializer(), serdes.topic()), + serdes.valueFrom(next.value)); } @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java index 01236ad3e09d2..657e05d11fa20 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java @@ -53,8 +53,10 @@ public void setUp() { stateDir = TestUtils.tempDirectory("kafka-test"); } - private void doTestKTable(final StreamsBuilder builder, final KTable table2, - final KTable table3, final String topic1) { + private void doTestKTable(final StreamsBuilder builder, + final KTable table2, + final KTable table3, + final String topic) { MockProcessorSupplier proc2 = new MockProcessorSupplier<>(); MockProcessorSupplier proc3 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); @@ -62,13 +64,13 @@ private void doTestKTable(final StreamsBuilder builder, final KTable KeyValueStore createKeyValueStore(ProcessorContext context, - Class keyClass, Class valueClass, - boolean useContextSerdes); + protected abstract KeyValueStore createKeyValueStore(final ProcessorContext context); protected MockProcessorContext context; protected KeyValueStore store; @@ -48,7 +51,7 @@ public void before() { driver = KeyValueStoreTestDriver.create(Integer.class, String.class); context = (MockProcessorContext) driver.context(); context.setTime(10); - store = createKeyValueStore(context, Integer.class, String.class, false); + store = createKeyValueStore(context); } @After @@ -67,6 +70,66 @@ private static Map getContents(final KeyValueIterator serializer = new StringSerializer() { + private int numCalls = 0; + + @Override + public byte[] serialize(final String topic, final String data) { + if (++numCalls > 3) { + fail("Value serializer is called; it should never happen"); + } + + return super.serialize(topic, data); + } + }; + + context.setValueSerde(Serdes.serdeFrom(serializer, new StringDeserializer())); + store = createKeyValueStore(driver.context()); + + store.put(0, "zero"); + store.put(1, "one"); + store.put(2, "two"); + store.delete(0); + store.delete(1); + + // should not include deleted records in iterator + final Map expectedContents = Collections.singletonMap(2, "two"); + assertEquals(expectedContents, getContents(store.all())); + } + + @Test + public void shouldDeleteIfSerializedValueIsNull() { + store.close(); + + final Serializer serializer = new StringSerializer() { + @Override + public byte[] serialize(final String topic, final String data) { + if (data.equals("null")) { + // will be serialized to null bytes, indicating deletes + return null; + } + return super.serialize(topic, data); + } + }; + + context.setValueSerde(Serdes.serdeFrom(serializer, new StringDeserializer())); + store = createKeyValueStore(driver.context()); + + store.put(0, "zero"); + store.put(1, "one"); + store.put(2, "two"); + store.put(0, "null"); + store.put(1, "null"); + + // should not include deleted records in iterator + final Map expectedContents = Collections.singletonMap(2, "two"); + assertEquals(expectedContents, getContents(store.all())); + } + @Test public void testPutGetRange() { // Verify that the store reads and writes correctly ... @@ -157,7 +220,7 @@ public void testRestore() { // Create the store, which should register with the context and automatically // receive the restore entries ... - store = createKeyValueStore(driver.context(), Integer.class, String.class, false); + store = createKeyValueStore(driver.context()); context.restore(store.name(), driver.restoredEntries()); // Verify that the store's contents were properly restored ... @@ -179,7 +242,7 @@ public void testRestoreWithDefaultSerdes() { // Create the store, which should register with the context and automatically // receive the restore entries ... - store = createKeyValueStore(driver.context(), Integer.class, String.class, true); + store = createKeyValueStore(driver.context()); context.restore(store.name(), driver.restoredEntries()); // Verify that the store's contents were properly restored ... assertEquals(0, driver.checkForRestoredEntries(store)); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java index 97a2fbf1d5b01..3ff343a7155d5 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.streams.state.internals; import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.LogContext; @@ -30,6 +31,7 @@ import org.apache.kafka.streams.processor.internals.RecordCollector; import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.Stores; import org.apache.kafka.test.MockProcessorContext; import org.junit.After; @@ -84,22 +86,14 @@ public void after() { @SuppressWarnings("unchecked") @Override - protected KeyValueStore createKeyValueStore(final ProcessorContext context, - final Class keyClass, - final Class valueClass, - final boolean useContextSerdes) { - final String storeName = "cache-store"; - - - final Stores.PersistentKeyValueFactory factory = Stores - .create(storeName) - .withKeys(Serdes.serdeFrom(keyClass)) - .withValues(Serdes.serdeFrom(valueClass)) - .persistent() - .enableCaching(); - - - final KeyValueStore store = (KeyValueStore) factory.build().get(); + protected KeyValueStore createKeyValueStore(final ProcessorContext context) { + final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore("cache-store"), + (Serde) context.keySerde(), + (Serde) context.valueSerde()) + .withCachingEnabled(); + + final KeyValueStore store = (KeyValueStore) storeBuilder.build(); final CacheFlushListenerStub cacheFlushListener = new CacheFlushListenerStub<>(); final CachedStateStore inner = (CachedStateStore) ((WrappedStateStore) store).wrappedStore(); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java index d24a90fd857de..adaab006cf5f1 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.streams.state.internals; import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.StateStore; @@ -36,27 +35,11 @@ public class InMemoryKeyValueLoggedStoreTest extends AbstractKeyValueStoreTest { @SuppressWarnings("unchecked") @Override - protected KeyValueStore createKeyValueStore( - ProcessorContext context, - Class keyClass, - Class valueClass, - boolean useContextSerdes) { - - final Serde keySerde; - final Serde valueSerde; - - if (useContextSerdes) { - keySerde = (Serde) context.keySerde(); - valueSerde = (Serde) context.valueSerde(); - } else { - keySerde = Serdes.serdeFrom(keyClass); - valueSerde = Serdes.serdeFrom(valueClass); - } - + protected KeyValueStore createKeyValueStore(final ProcessorContext context) { final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder( Stores.inMemoryKeyValueStore("my-store"), - keySerde, - valueSerde) + (Serde) context.keySerde(), + (Serde) context.valueSerde()) .withLoggingEnabled(Collections.singletonMap("retention.ms", "1000")); final StateStore store = storeBuilder.build(); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java index 541c0038261f1..ef5d6dcfb5aa7 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java @@ -16,9 +16,11 @@ */ package org.apache.kafka.streams.state.internals; +import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStoreSupplier; +import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.Stores; import org.junit.Test; @@ -30,22 +32,15 @@ public class InMemoryKeyValueStoreTest extends AbstractKeyValueStoreTest { @SuppressWarnings("unchecked") @Override - protected KeyValueStore createKeyValueStore( - ProcessorContext context, - Class keyClass, - Class valueClass, - boolean useContextSerdes) { + protected KeyValueStore createKeyValueStore(final ProcessorContext context) { + final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder( + Stores.inMemoryKeyValueStore("my-store"), + (Serde) context.keySerde(), + (Serde) context.valueSerde()); - StateStoreSupplier supplier; - if (useContextSerdes) { - supplier = Stores.create("my-store").withKeys(context.keySerde()).withValues(context.valueSerde()).inMemory().build(); - } else { - supplier = Stores.create("my-store").withKeys(keyClass).withValues(valueClass).inMemory().build(); - } - - KeyValueStore store = (KeyValueStore) supplier.get(); + final StateStore store = storeBuilder.build(); store.init(context, store); - return store; + return (KeyValueStore) store; } @Test @@ -59,7 +54,7 @@ public void shouldRemoveKeysWithNullValues() { driver.addEntryToRestoreLog(3, "three"); driver.addEntryToRestoreLog(0, null); - store = createKeyValueStore(driver.context(), Integer.class, String.class, true); + store = createKeyValueStore(driver.context()); context.restore(store.name(), driver.restoredEntries()); assertEquals(3, driver.sizeOf(store)); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java index 7dda58552f36a..2d39ae72ab63c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java @@ -16,10 +16,12 @@ */ package org.apache.kafka.streams.state.internals; +import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStoreSupplier; +import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.Stores; import org.junit.Test; @@ -36,22 +38,17 @@ public class InMemoryLRUCacheStoreTest extends AbstractKeyValueStoreTest { @SuppressWarnings("unchecked") @Override - protected KeyValueStore createKeyValueStore( - ProcessorContext context, - Class keyClass, - Class valueClass, - boolean useContextSerdes) { - - StateStoreSupplier supplier; - if (useContextSerdes) { - supplier = Stores.create("my-store").withKeys(context.keySerde()).withValues(context.valueSerde()).inMemory().maxEntries(10).build(); - } else { - supplier = Stores.create("my-store").withKeys(keyClass).withValues(valueClass).inMemory().maxEntries(10).build(); - } + protected KeyValueStore createKeyValueStore(final ProcessorContext context) { + + final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder( + Stores.lruMap("my-store", 10), + (Serde) context.keySerde(), + (Serde) context.valueSerde()); - KeyValueStore store = (KeyValueStore) supplier.get(); + final StateStore store = storeBuilder.build(); store.init(context, store); - return store; + + return (KeyValueStore) store; } @Test @@ -157,7 +154,7 @@ public void testRestoreEvict() { // Create the store, which should register with the context and automatically // receive the restore entries ... - store = createKeyValueStore(driver.context(), Integer.class, String.class, false); + store = createKeyValueStore(driver.context()); context.restore(store.name(), driver.restoredEntries()); // Verify that the store's changelog does not get more appends ... assertEquals(0, driver.numFlushedEntryStored()); diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java index 5aaf82f74c80f..ec2e3383acaad 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreTest.java @@ -16,11 +16,14 @@ */ package org.apache.kafka.streams.state.internals; +import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.streams.errors.InvalidStateStoreException; import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.RocksDBConfigSetter; +import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.Stores; import org.junit.Test; import org.rocksdb.Options; @@ -36,29 +39,15 @@ public class RocksDBKeyValueStoreTest extends AbstractKeyValueStoreTest { @SuppressWarnings("unchecked") @Override - protected KeyValueStore createKeyValueStore(final ProcessorContext context, - final Class keyClass, - final Class valueClass, - final boolean useContextSerdes) { - final Stores.PersistentKeyValueFactory factory; - if (useContextSerdes) { - factory = Stores - .create("my-store") - .withKeys(context.keySerde()) - .withValues(context.valueSerde()) - .persistent(); - - } else { - factory = Stores - .create("my-store") - .withKeys(keyClass) - .withValues(valueClass) - .persistent(); - } + protected KeyValueStore createKeyValueStore(final ProcessorContext context) { + final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore("my-store"), + (Serde) context.keySerde(), + (Serde) context.valueSerde()); - final KeyValueStore store = (KeyValueStore) factory.build().get(); + final StateStore store = storeBuilder.build(); store.init(context, store); - return store; + return (KeyValueStore) store; } public static class TheRocksDbConfigSetter implements RocksDBConfigSetter { diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIteratorTest.java deleted file mode 100644 index d11b9b4498477..0000000000000 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SerializedKeyValueIteratorTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.streams.state.internals; - -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.utils.Bytes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.state.StateSerdes; -import org.apache.kafka.test.KeyValueIteratorStub; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Iterator; -import java.util.NoSuchElementException; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class SerializedKeyValueIteratorTest { - - private final StateSerdes serdes = new StateSerdes<>("blah", Serdes.String(), Serdes.String()); - private final Iterator> iterator - = Arrays.asList(KeyValue.pair(Bytes.wrap("hi".getBytes()), "there".getBytes()), - KeyValue.pair(Bytes.wrap("hello".getBytes()), "world".getBytes())) - .iterator(); - private final DelegatingPeekingKeyValueIterator peeking - = new DelegatingPeekingKeyValueIterator<>("store", new KeyValueIteratorStub<>(iterator)); - private final SerializedKeyValueIterator serializedKeyValueIterator - = new SerializedKeyValueIterator<>(peeking, serdes); - - @Test - public void shouldReturnTrueOnHasNextWhenMoreResults() { - assertTrue(serializedKeyValueIterator.hasNext()); - } - - @Test - public void shouldReturnNextValueWhenItExists() { - assertThat(serializedKeyValueIterator.next(), equalTo(KeyValue.pair("hi", "there"))); - assertThat(serializedKeyValueIterator.next(), equalTo(KeyValue.pair("hello", "world"))); - } - - @Test - public void shouldReturnFalseOnHasNextWhenNoMoreResults() { - advanceIteratorToEnd(); - assertFalse(serializedKeyValueIterator.hasNext()); - } - - @Test - public void shouldThrowNoSuchElementOnNextWhenIteratorExhausted() { - advanceIteratorToEnd(); - try { - serializedKeyValueIterator.next(); - fail("Expected NoSuchElementException on exhausted iterator"); - } catch (final NoSuchElementException nse) { - // pass - } - } - - @Test - public void shouldPeekNextKey() { - assertThat(serializedKeyValueIterator.peekNextKey(), equalTo("hi")); - serializedKeyValueIterator.next(); - assertThat(serializedKeyValueIterator.peekNextKey(), equalTo("hello")); - } - - @Test(expected = UnsupportedOperationException.class) - public void shouldThrowUnsupportedOperationOnRemove() { - serializedKeyValueIterator.remove(); - } - - private void advanceIteratorToEnd() { - serializedKeyValueIterator.next(); - serializedKeyValueIterator.next(); - } - - -} \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java index ce6cca8e45151..06137fbca7a6f 100644 --- a/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java +++ b/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java @@ -50,12 +50,12 @@ public class MockProcessorContext extends AbstractProcessorContext implements Re private final File stateDir; private final Metrics metrics; - private final Serde keySerde; - private final Serde valSerde; private final RecordCollector.Supplier recordCollectorSupplier; private final Map storeMap = new LinkedHashMap<>(); private final Map restoreFuncs = new HashMap<>(); + private Serde keySerde; + private Serde valSerde; private long timestamp = -1L; public MockProcessorContext(final File stateDir, @@ -121,6 +121,14 @@ public RecordCollector recordCollector() { return recordCollector; } + public void setKeySerde(final Serde keySerde) { + this.keySerde = keySerde; + } + + public void setValueSerde(final Serde valSerde) { + this.valSerde = valSerde; + } + // serdes will override whatever specified in the configs @Override public Serde keySerde() { From caf03d642b6992bf95b922f0cb5f724d265d4029 Mon Sep 17 00:00:00 2001 From: Jun Rao Date: Tue, 6 Feb 2018 14:04:46 -0800 Subject: [PATCH 0014/1847] Minor: fix compilation error in KAFKA-6254 --- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index e81e3d1fc3ad1..de621b41356da 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2183,7 +2183,7 @@ private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Er long lastStableOffset, long logStartOffset, int throttleTime) { Map partitions = Collections.singletonMap(tp, new FetchResponse.PartitionData(error, hw, lastStableOffset, logStartOffset, null, records)); - return new FetchResponse(new LinkedHashMap<>(partitions), throttleTime); + return new FetchResponse(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); } private MetadataResponse newMetadataResponse(String topic, Errors error) { From 65b5ccf6413369aa4f21c72abcdf31ca72a79b00 Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Wed, 7 Feb 2018 03:29:08 -0800 Subject: [PATCH 0015/1847] KAFKA-6532: Reduce impact of delegation tokens on public interfaces (#4524) Keep delegation token implementation internal without exposing implementation details to pluggable classes: 1. KafkaPrincipal#tokenAuthenticated must always be set by SaslServerAuthenticator so that custom PrincipalBuilders cannot override. 2. Replace o.a.k.c.security.scram.DelegationTokenAuthenticationCallback with a more generic ScramExtensionsCallback that can be used to add more extensions in future. 3. Separate out ScramCredentialCallback (KIP-86 makes this a public interface) from delegation token credential callback (which is internal). Reviewers: Jun Rao , Manikumar Reddy --- .../common/security/auth/KafkaPrincipal.java | 12 ++- .../DefaultKafkaPrincipalBuilder.java | 10 +-- .../SaslClientCallbackHandler.java | 10 +-- .../SaslServerAuthenticator.java | 7 +- .../scram/ScramCredentialCallback.java | 24 ------ .../security/scram/ScramExtensions.java | 78 +++++++++++++++++++ ...back.java => ScramExtensionsCallback.java} | 14 ++-- .../security/scram/ScramLoginModule.java | 6 +- .../common/security/scram/ScramMessages.java | 31 +++----- .../security/scram/ScramSaslClient.java | 8 +- .../security/scram/ScramSaslServer.java | 43 ++++++---- .../scram/ScramServerCallbackHandler.java | 15 ++-- .../DelegationTokenCredentialCallback.java | 31 ++++++++ .../DefaultKafkaPrincipalBuilderTest.java | 2 - .../security/scram/ScramMessagesTest.java | 6 +- 15 files changed, 193 insertions(+), 104 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java rename clients/src/main/java/org/apache/kafka/common/security/scram/{DelegationTokenAuthenticationCallback.java => ScramExtensionsCallback.java} (72%) create mode 100644 clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java diff --git a/clients/src/main/java/org/apache/kafka/common/security/auth/KafkaPrincipal.java b/clients/src/main/java/org/apache/kafka/common/security/auth/KafkaPrincipal.java index 10bf76dd2e0c2..74bc9c916c081 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/auth/KafkaPrincipal.java +++ b/clients/src/main/java/org/apache/kafka/common/security/auth/KafkaPrincipal.java @@ -47,16 +47,11 @@ public class KafkaPrincipal implements Principal { private final String principalType; private final String name; - private boolean tokenAuthenticated; + private volatile boolean tokenAuthenticated; public KafkaPrincipal(String principalType, String name) { - this(principalType, name, false); - } - - public KafkaPrincipal(String principalType, String name, boolean tokenauth) { this.principalType = requireNonNull(principalType, "Principal type cannot be null"); this.name = requireNonNull(name, "Principal name cannot be null"); - this.tokenAuthenticated = tokenauth; } /** @@ -91,7 +86,6 @@ public boolean equals(Object o) { public int hashCode() { int result = principalType != null ? principalType.hashCode() : 0; result = 31 * result + (name != null ? name.hashCode() : 0); - result = 31 * result + (tokenAuthenticated ? 1 : 0); return result; } @@ -104,6 +98,10 @@ public String getPrincipalType() { return principalType; } + public void tokenAuthenticated(boolean tokenAuthenticated) { + this.tokenAuthenticated = tokenAuthenticated; + } + public boolean tokenAuthenticated() { return tokenAuthenticated; } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/DefaultKafkaPrincipalBuilder.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/DefaultKafkaPrincipalBuilder.java index 740423863af9a..30b0a3e0980f0 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/DefaultKafkaPrincipalBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/DefaultKafkaPrincipalBuilder.java @@ -28,7 +28,6 @@ import org.apache.kafka.common.security.auth.SslAuthenticationContext; import org.apache.kafka.common.security.kerberos.KerberosName; import org.apache.kafka.common.security.kerberos.KerberosShortNamer; -import org.apache.kafka.common.security.scram.ScramLoginModule; import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; @@ -119,13 +118,8 @@ public KafkaPrincipal build(AuthenticationContext context) { SaslServer saslServer = ((SaslAuthenticationContext) context).server(); if (SaslConfigs.GSSAPI_MECHANISM.equals(saslServer.getMechanismName())) return applyKerberosShortNamer(saslServer.getAuthorizationID()); - else { - Boolean isTokenAuthenticated = (Boolean) saslServer.getNegotiatedProperty(ScramLoginModule.TOKEN_AUTH_CONFIG); - if (isTokenAuthenticated != null && isTokenAuthenticated) - return new KafkaPrincipal(KafkaPrincipal.USER_TYPE, saslServer.getAuthorizationID(), true); - else - return new KafkaPrincipal(KafkaPrincipal.USER_TYPE, saslServer.getAuthorizationID()); - } + else + return new KafkaPrincipal(KafkaPrincipal.USER_TYPE, saslServer.getAuthorizationID()); } else { throw new IllegalArgumentException("Unhandled authentication context type: " + context.getClass().getName()); } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java index de96cef1d4315..31c51c22ca38c 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java @@ -28,7 +28,7 @@ import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.network.Mode; -import org.apache.kafka.common.security.scram.DelegationTokenAuthenticationCallback; +import org.apache.kafka.common.security.scram.ScramExtensionsCallback; /** * Callback handler for Sasl clients. The callbacks required for the SASL mechanism @@ -81,10 +81,10 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException { ac.setAuthorized(authId.equals(authzId)); if (ac.isAuthorized()) ac.setAuthorizedID(authzId); - } else if (callback instanceof DelegationTokenAuthenticationCallback) { - DelegationTokenAuthenticationCallback tc = (DelegationTokenAuthenticationCallback) callback; - if (!isKerberos && subject != null && !subject.getPublicCredentials(Boolean.class).isEmpty()) { - tc.tokenauth(subject.getPublicCredentials(Boolean.class).iterator().next()); + } else if (callback instanceof ScramExtensionsCallback) { + ScramExtensionsCallback sc = (ScramExtensionsCallback) callback; + if (!isKerberos && subject != null && !subject.getPublicCredentials(Map.class).isEmpty()) { + sc.extensions((Map) subject.getPublicCredentials(Map.class).iterator().next()); } } else { throw new UnsupportedCallbackException(callback, "Unrecognized SASL ClientCallback"); diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index ca6e9d2c21a7f..2a80e5bc0e4c0 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -53,6 +53,7 @@ import org.apache.kafka.common.security.kerberos.KerberosName; import org.apache.kafka.common.security.kerberos.KerberosShortNamer; import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.ScramLoginModule; import org.apache.kafka.common.security.scram.ScramMechanism; import org.apache.kafka.common.security.scram.ScramServerCallbackHandler; import org.apache.kafka.common.utils.Utils; @@ -297,7 +298,11 @@ public void authenticate() throws IOException { @Override public KafkaPrincipal principal() { SaslAuthenticationContext context = new SaslAuthenticationContext(saslServer, securityProtocol, clientAddress()); - return principalBuilder.build(context); + KafkaPrincipal principal = principalBuilder.build(context); + if (ScramMechanism.isScram(saslMechanism) && Boolean.parseBoolean((String) saslServer.getNegotiatedProperty(ScramLoginModule.TOKEN_AUTH_CONFIG))) { + principal.tokenAuthenticated(true); + } + return principal; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java index 7f3601cf24cfd..931210a0e0702 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java @@ -20,36 +20,12 @@ public class ScramCredentialCallback implements Callback { private ScramCredential scramCredential; - private boolean tokenAuthenticated; - private String tokenOwner; - private String mechanism; - - public ScramCredentialCallback(boolean tokenAuthenticated, String mechanism) { - this.tokenAuthenticated = tokenAuthenticated; - this.mechanism = mechanism; - } public ScramCredential scramCredential() { return scramCredential; } - public boolean tokenauth() { - return tokenAuthenticated; - } - public void scramCredential(ScramCredential scramCredential) { this.scramCredential = scramCredential; } - - public void tokenOwner(String tokenOwner) { - this.tokenOwner = tokenOwner; - } - - public String tokenOwner() { - return tokenOwner; - } - - public String mechanism() { - return mechanism; - } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java new file mode 100644 index 0000000000000..0f461c0e3c2df --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.security.scram; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class ScramExtensions { + private final Map extensionMap; + + public ScramExtensions() { + this(Collections.emptyMap()); + } + + public ScramExtensions(String extensions) { + this(stringToMap(extensions)); + } + + public ScramExtensions(Map extensionMap) { + this.extensionMap = extensionMap; + } + + public String extensionValue(String name) { + return extensionMap.get(name); + } + + public Set extensionNames() { + return extensionMap.keySet(); + } + + public boolean tokenAuthenticated() { + return Boolean.parseBoolean(extensionMap.get(ScramLoginModule.TOKEN_AUTH_CONFIG)); + } + + @Override + public String toString() { + return mapToString(extensionMap); + } + + private static Map stringToMap(String extensions) { + Map extensionMap = new HashMap<>(); + + if (!extensions.isEmpty()) { + String[] attrvals = extensions.split(","); + for (String attrval : attrvals) { + String[] array = attrval.split("=", 2); + extensionMap.put(array[0], array[1]); + } + } + return extensionMap; + } + + private static String mapToString(Map extensionMap) { + StringBuilder builder = new StringBuilder(); + for (Map.Entry entry : extensionMap.entrySet()) { + builder.append(entry.getKey()); + builder.append('='); + builder.append(entry.getValue()); + } + return builder.toString(); + } +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/DelegationTokenAuthenticationCallback.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java similarity index 72% rename from clients/src/main/java/org/apache/kafka/common/security/scram/DelegationTokenAuthenticationCallback.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java index df6e849309a92..b40468bd650ef 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/DelegationTokenAuthenticationCallback.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java @@ -18,15 +18,17 @@ package org.apache.kafka.common.security.scram; import javax.security.auth.callback.Callback; +import java.util.Collections; +import java.util.Map; -public class DelegationTokenAuthenticationCallback implements Callback { - private boolean tokenauth; +public class ScramExtensionsCallback implements Callback { + private Map extensions = Collections.emptyMap(); - public String extension() { - return ScramLoginModule.TOKEN_AUTH_CONFIG + "=" + Boolean.toString(tokenauth); + public Map extensions() { + return extensions; } - public void tokenauth(Boolean tokenauth) { - this.tokenauth = tokenauth; + public void extensions(Map extensions) { + this.extensions = extensions; } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java index 8000f4c7f2a13..43df515256fc9 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.common.security.scram; +import java.util.Collections; import java.util.Map; import javax.security.auth.Subject; @@ -44,7 +45,10 @@ public void initialize(Subject subject, CallbackHandler callbackHandler, Map scramExtensions = Collections.singletonMap(TOKEN_AUTH_CONFIG, "true"); + subject.getPublicCredentials().add(scramExtensions); + } } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java index e697ea5f7ba60..05b3d7755404e 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.utils.Base64; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -77,7 +76,7 @@ public static class ClientFirstMessage extends AbstractScramMessage { private final String saslName; private final String nonce; private final String authorizationId; - private final String extensions; + private final ScramExtensions extensions; public ClientFirstMessage(byte[] messageBytes) throws SaslException { String message = toMessage(messageBytes); Matcher matcher = PATTERN.matcher(message); @@ -88,12 +87,13 @@ public ClientFirstMessage(byte[] messageBytes) throws SaslException { this.saslName = matcher.group("saslname"); this.nonce = matcher.group("nonce"); String extString = matcher.group("extensions"); - this.extensions = extString.startsWith(",") ? extString.substring(1) : extString; + + this.extensions = extString.startsWith(",") ? new ScramExtensions(extString.substring(1)) : new ScramExtensions(); } - public ClientFirstMessage(String saslName, String nonce, String extensions) { + public ClientFirstMessage(String saslName, String nonce, Map extensions) { this.saslName = saslName; this.nonce = nonce; - this.extensions = extensions; + this.extensions = new ScramExtensions(extensions); this.authorizationId = ""; // Optional authzid not specified in gs2-header } public String saslName() { @@ -108,29 +108,16 @@ public String authorizationId() { public String gs2Header() { return "n," + authorizationId + ","; } - public String extensions() { + public ScramExtensions extensions() { return extensions; } - public Map extensionsAsMap() { - Map map = new HashMap<>(); - - if (extensions.isEmpty()) - return map; - - String[] attrvals = extensions.split(","); - for (String attrval: attrvals) { - String[] array = attrval.split("=", 2); - map.put(array[0], array[1]); - } - return map; - } - public String clientFirstMessageBare() { - if (extensions.isEmpty()) + String extensionStr = extensions.toString(); + if (extensionStr.isEmpty()) return String.format("n=%s,r=%s", saslName, nonce); else - return String.format("n=%s,r=%s,%s", saslName, nonce, extensions); + return String.format("n=%s,r=%s,%s", saslName, nonce, extensionStr); } String toMessage() { return gs2Header() + clientFirstMessageBare(); diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java index 6b66f5ece9915..71109df2ddd1f 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java @@ -97,18 +97,18 @@ public byte[] evaluateChallenge(byte[] challenge) throws SaslException { throw new SaslException("Expected empty challenge"); clientNonce = formatter.secureRandomString(); NameCallback nameCallback = new NameCallback("Name:"); - DelegationTokenAuthenticationCallback tokenAuthCallback = new DelegationTokenAuthenticationCallback(); + ScramExtensionsCallback extensionsCallback = new ScramExtensionsCallback(); try { - callbackHandler.handle(new Callback[]{nameCallback, tokenAuthCallback}); + callbackHandler.handle(new Callback[]{nameCallback, extensionsCallback}); } catch (IOException | UnsupportedCallbackException e) { throw new SaslException("User name could not be obtained", e); } String username = nameCallback.getName(); String saslName = formatter.saslName(username); - String extension = tokenAuthCallback.extension(); - this.clientFirstMessage = new ScramMessages.ClientFirstMessage(saslName, clientNonce, extension); + Map extensions = extensionsCallback.extensions(); + this.clientFirstMessage = new ScramMessages.ClientFirstMessage(saslName, clientNonce, extensions); setState(State.RECEIVE_SERVER_FIRST_MESSAGE); return clientFirstMessage.toBytes(); diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java index 94b92b6e1ea7e..314c1d413ba52 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.Set; import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; @@ -37,6 +38,8 @@ import org.apache.kafka.common.security.scram.ScramMessages.ClientFirstMessage; import org.apache.kafka.common.security.scram.ScramMessages.ServerFinalMessage; import org.apache.kafka.common.security.scram.ScramMessages.ServerFirstMessage; +import org.apache.kafka.common.security.token.delegation.DelegationTokenCredentialCallback; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +53,7 @@ public class ScramSaslServer implements SaslServer { private static final Logger log = LoggerFactory.getLogger(ScramSaslServer.class); + private static final Set SUPPORTED_EXTENSIONS = Utils.mkSet(ScramLoginModule.TOKEN_AUTH_CONFIG); enum State { RECEIVE_CLIENT_FIRST_MESSAGE, @@ -65,9 +69,9 @@ enum State { private String username; private ClientFirstMessage clientFirstMessage; private ServerFirstMessage serverFirstMessage; + private ScramExtensions scramExtensions; private ScramCredential scramCredential; - private boolean tokenAuthentication; - private String tokenOwner; + private String authorizationId; public ScramSaslServer(ScramMechanism mechanism, Map props, CallbackHandler callbackHandler) throws NoSuchAlgorithmException { this.mechanism = mechanism; @@ -91,18 +95,29 @@ public byte[] evaluateResponse(byte[] response) throws SaslException, SaslAuthen switch (state) { case RECEIVE_CLIENT_FIRST_MESSAGE: this.clientFirstMessage = new ClientFirstMessage(response); + this.scramExtensions = clientFirstMessage.extensions(); + if (!SUPPORTED_EXTENSIONS.containsAll(scramExtensions.extensionNames())) { + log.debug("Unsupported extensions will be ignored, supported {}, provided {}", + SUPPORTED_EXTENSIONS, scramExtensions.extensionNames()); + } String serverNonce = formatter.secureRandomString(); try { String saslName = clientFirstMessage.saslName(); this.username = formatter.username(saslName); - Map extensions = clientFirstMessage.extensionsAsMap(); - this.tokenAuthentication = "true".equalsIgnoreCase(extensions.get(ScramLoginModule.TOKEN_AUTH_CONFIG)); NameCallback nameCallback = new NameCallback("username", username); - ScramCredentialCallback credentialCallback = new ScramCredentialCallback(tokenAuthentication, getMechanismName()); - callbackHandler.handle(new Callback[]{nameCallback, credentialCallback}); - this.tokenOwner = credentialCallback.tokenOwner(); - if (tokenAuthentication && tokenOwner == null) - throw new SaslException("Token Authentication failed: Invalid tokenId : " + username); + ScramCredentialCallback credentialCallback; + if (scramExtensions.tokenAuthenticated()) { + DelegationTokenCredentialCallback tokenCallback = new DelegationTokenCredentialCallback(); + credentialCallback = tokenCallback; + callbackHandler.handle(new Callback[]{nameCallback, tokenCallback}); + if (tokenCallback.tokenOwner() == null) + throw new SaslException("Token Authentication failed: Invalid tokenId : " + username); + this.authorizationId = tokenCallback.tokenOwner(); + } else { + credentialCallback = new ScramCredentialCallback(); + callbackHandler.handle(new Callback[]{nameCallback, credentialCallback}); + this.authorizationId = username; + } this.scramCredential = credentialCallback.scramCredential(); if (scramCredential == null) throw new SaslException("Authentication failed: Invalid user credentials"); @@ -150,11 +165,7 @@ public byte[] evaluateResponse(byte[] response) throws SaslException, SaslAuthen public String getAuthorizationID() { if (!isComplete()) throw new IllegalStateException("Authentication exchange has not completed"); - - if (tokenAuthentication) - return tokenOwner; // return token owner as principal for this session - - return username; + return authorizationId; } @Override @@ -167,8 +178,8 @@ public Object getNegotiatedProperty(String propName) { if (!isComplete()) throw new IllegalStateException("Authentication exchange has not completed"); - if (ScramLoginModule.TOKEN_AUTH_CONFIG.equals(propName)) - return tokenAuthentication; + if (SUPPORTED_EXTENSIONS.contains(propName)) + return scramExtensions.extensionValue(propName); else return null; } diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java index a064e8a3423f3..5e37eae9d4fb6 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java @@ -28,11 +28,13 @@ import org.apache.kafka.common.security.authenticator.AuthCallbackHandler; import org.apache.kafka.common.security.authenticator.CredentialCache; import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; +import org.apache.kafka.common.security.token.delegation.DelegationTokenCredentialCallback; public class ScramServerCallbackHandler implements AuthCallbackHandler { private final CredentialCache.Cache credentialCache; private final DelegationTokenCache tokenCache; + private String saslMechanism; public ScramServerCallbackHandler(CredentialCache.Cache credentialCache, DelegationTokenCache tokenCache) { @@ -46,13 +48,13 @@ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallback for (Callback callback : callbacks) { if (callback instanceof NameCallback) username = ((NameCallback) callback).getDefaultName(); - else if (callback instanceof ScramCredentialCallback) { + else if (callback instanceof DelegationTokenCredentialCallback) { + DelegationTokenCredentialCallback tokenCallback = (DelegationTokenCredentialCallback) callback; + tokenCallback.scramCredential(tokenCache.credential(saslMechanism, username)); + tokenCallback.tokenOwner(tokenCache.owner(username)); + } else if (callback instanceof ScramCredentialCallback) { ScramCredentialCallback sc = (ScramCredentialCallback) callback; - if (sc.tokenauth()) { - sc.scramCredential(tokenCache.credential(sc.mechanism(), username)); - sc.tokenOwner(tokenCache.owner(username)); - } else - sc.scramCredential(credentialCache.get(username)); + sc.scramCredential(credentialCache.get(username)); } else throw new UnsupportedCallbackException(callback); } @@ -60,6 +62,7 @@ else if (callback instanceof ScramCredentialCallback) { @Override public void configure(Map configs, Mode mode, Subject subject, String saslMechanism) { + this.saslMechanism = saslMechanism; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java new file mode 100644 index 0000000000000..7490a3e91b1ca --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.security.token.delegation; + +import org.apache.kafka.common.security.scram.ScramCredentialCallback; + +public class DelegationTokenCredentialCallback extends ScramCredentialCallback { + private String tokenOwner; + + public void tokenOwner(String tokenOwner) { + this.tokenOwner = tokenOwner; + } + + public String tokenOwner() { + return tokenOwner; + } +} \ No newline at end of file diff --git a/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java index 787f5a72cd7a7..a30c09ff3167e 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder; import org.apache.kafka.common.security.kerberos.KerberosName; import org.apache.kafka.common.security.kerberos.KerberosShortNamer; -import org.apache.kafka.common.security.scram.ScramLoginModule; import org.apache.kafka.common.security.scram.ScramMechanism; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; @@ -119,7 +118,6 @@ public void testPrincipalBuilderScram() throws Exception { EasyMock.expect(server.getMechanismName()).andReturn(ScramMechanism.SCRAM_SHA_256.mechanismName()); EasyMock.expect(server.getAuthorizationID()).andReturn("foo"); - EasyMock.expect(server.getNegotiatedProperty(ScramLoginModule.TOKEN_AUTH_CONFIG)).andReturn(false); replayAll(); diff --git a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java b/clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java index 58be6e19bf602..7b04ede6e786a 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java @@ -21,12 +21,14 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; +import java.util.Collections; import javax.security.sasl.SaslException; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.apache.kafka.common.security.scram.ScramMessages.AbstractScramMessage; @@ -70,7 +72,7 @@ public void setUp() throws Exception { @Test public void validClientFirstMessage() throws SaslException { String nonce = formatter.secureRandomString(); - ClientFirstMessage m = new ClientFirstMessage("someuser", nonce, ""); + ClientFirstMessage m = new ClientFirstMessage("someuser", nonce, Collections.emptyMap()); checkClientFirstMessage(m, "someuser", nonce, ""); // Default format used by Kafka client: only user and nonce are specified @@ -111,7 +113,7 @@ public void validClientFirstMessage() throws SaslException { //optional tokenauth specified as extensions str = String.format("n,,n=testuser,r=%s,%s", nonce, "tokenauth=true"); m = createScramMessage(ClientFirstMessage.class, str); - assertEquals("tokenauth=true", m.extensions()); + assertTrue("Token authentication not set from extensions", m.extensions().tokenAuthenticated()); } @Test From 263a510676ee0173cbaa11b5de1de4c19d2f5202 Mon Sep 17 00:00:00 2001 From: Bill Bejeck Date: Wed, 7 Feb 2018 14:07:32 -0500 Subject: [PATCH 0016/1847] KAFKA-6367: StateRestoreListener use actual last restored offset for restored batch (#4507) Author: Bill Bejeck Reviewers: Guozhang Wang , Matthias J. Sax --- .../processor/StateRestoreListener.java | 4 ++-- .../internals/StoreChangelogReader.java | 5 ++-- .../internals/StoreChangelogReaderTest.java | 24 +++++++++++++++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreListener.java b/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreListener.java index c80a736734d7d..ea1c2888409d2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreListener.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/StateRestoreListener.java @@ -43,7 +43,7 @@ public interface StateRestoreListener { * @param topicPartition the TopicPartition containing the values to restore * @param storeName the name of the store undergoing restoration * @param startingOffset the starting offset of the entire restoration process for this TopicPartition - * @param endingOffset the ending offset of the entire restoration process for this TopicPartition + * @param endingOffset the exclusive ending offset of the entire restoration process for this TopicPartition */ void onRestoreStart(final TopicPartition topicPartition, final String storeName, @@ -62,7 +62,7 @@ void onRestoreStart(final TopicPartition topicPartition, * * @param topicPartition the TopicPartition containing the values to restore * @param storeName the name of the store undergoing restoration - * @param batchEndOffset the ending offset for the current restored batch for this TopicPartition + * @param batchEndOffset the inclusive ending offset for the current restored batch for this TopicPartition * @param numRestored the total number of records restored in this batch for this TopicPartition */ void onBatchRestored(final TopicPartition topicPartition, diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java index ba17ce95ede0b..b11c45ba313b3 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java @@ -273,12 +273,14 @@ private long processNext(final List> records, long nextPosition = -1; int numberRecords = records.size(); int numberRestored = 0; + long lastRestoredOffset = -1; for (final ConsumerRecord record : records) { final long offset = record.offset(); if (restorer.hasCompleted(offset, endOffset)) { nextPosition = record.offset(); break; } + lastRestoredOffset = offset; numberRestored++; if (record.key() != null) { restoreRecords.add(KeyValue.pair(record.key(), record.value())); @@ -295,8 +297,7 @@ private long processNext(final List> records, if (!restoreRecords.isEmpty()) { restorer.restore(restoreRecords); - restorer.restoreBatchCompleted(nextPosition, records.size()); - + restorer.restoreBatchCompleted(lastRestoredOffset, records.size()); } return nextPosition; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java index ee9645134150e..e69cede23fd22 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java @@ -234,15 +234,28 @@ public void shouldRestoreAndNotifyMultipleStores() throws Exception { assertThat(callbackTwo.restored.size(), equalTo(3)); assertAllCallbackStatesExecuted(callback, "storeName1"); - assertCorrectOffsetsReportedByListener(callback, 0L, 10L, 10L); + assertCorrectOffsetsReportedByListener(callback, 0L, 9L, 10L); assertAllCallbackStatesExecuted(callbackOne, "storeName2"); - assertCorrectOffsetsReportedByListener(callbackOne, 0L, 5L, 5L); + assertCorrectOffsetsReportedByListener(callbackOne, 0L, 4L, 5L); assertAllCallbackStatesExecuted(callbackTwo, "storeName3"); - assertCorrectOffsetsReportedByListener(callbackTwo, 0L, 3L, 3L); + assertCorrectOffsetsReportedByListener(callbackTwo, 0L, 2L, 3L); } + @Test + public void shouldOnlyReportTheLastRestoredOffset() { + setupConsumer(10, topicPartition); + changelogReader + .register(new StateRestorer(topicPartition, restoreListener, null, 5, true, "storeName1")); + changelogReader.restore(active); + + assertThat(callback.restored.size(), equalTo(5)); + assertAllCallbackStatesExecuted(callback, "storeName1"); + assertCorrectOffsetsReportedByListener(callback, 0L, 4L, 5L); + } + + private void assertAllCallbackStatesExecuted(final MockStateRestoreListener restoreListener, final String storeName) { assertThat(restoreListener.storeNameCalledStates.get(RESTORE_START), equalTo(storeName)); @@ -253,11 +266,12 @@ private void assertAllCallbackStatesExecuted(final MockStateRestoreListener rest private void assertCorrectOffsetsReportedByListener(final MockStateRestoreListener restoreListener, final long startOffset, - final long batchOffset, final long endOffset) { + final long batchOffset, + final long totalRestored) { assertThat(restoreListener.restoreStartOffset, equalTo(startOffset)); assertThat(restoreListener.restoredBatchOffset, equalTo(batchOffset)); - assertThat(restoreListener.restoreEndOffset, equalTo(endOffset)); + assertThat(restoreListener.totalNumRestored, equalTo(totalRestored)); } @Test From 164fea30ea6b43d65074957f1f7b8342721b2a31 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Wed, 7 Feb 2018 11:43:57 -0800 Subject: [PATCH 0017/1847] MINOR: update upgrade notes for Streams API; message format 0.10 requiered (#4500) Author: Matthias J. Sax Reviewers: Bill Bejeck , Guozhang Wang --- docs/streams/developer-guide/datatypes.html | 2 +- docs/streams/developer-guide/write-streams.html | 10 +++++----- docs/upgrade.html | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/streams/developer-guide/datatypes.html b/docs/streams/developer-guide/datatypes.html index 155ee2cc65bb0..a86ea3e5790ef 100644 --- a/docs/streams/developer-guide/datatypes.html +++ b/docs/streams/developer-guide/datatypes.html @@ -102,7 +102,7 @@

    Primitive and basic types
    <dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka-clients</artifactId>
-    <version>1.0.0-cp1</version>
+    <version>{{fullDotVersion}}</version>
 </dependency>
    diff --git a/docs/streams/developer-guide/write-streams.html b/docs/streams/developer-guide/write-streams.html index c9ca49ca0792b..1e4213d1b81a7 100644 --- a/docs/streams/developer-guide/write-streams.html +++ b/docs/streams/developer-guide/write-streams.html @@ -69,12 +69,12 @@ org.apache.kafka kafka-streams - 1.0.0 + {{fullDotVersion}} (Required) Base library for Kafka Streams. org.apache.kafka kafka-clients - 1.0.0 + {{fullDotVersion}} (Required) Kafka client library. Contains built-in serializers/deserializers. @@ -87,12 +87,12 @@ 
    <dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka-streams</artifactId>
-    <version>1.0.0</version>
+    <version>{{fullDotVersion}}</version>
 </dependency>
 <dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka-clients</artifactId>
-    <version>1.0.0</version>
+    <version>{{fullDotVersion}}</version>
 </dependency>
    @@ -245,4 +245,4 @@

    Using Kafka Streams within your application codeNew Prot
    Upgrading a 1.0.0 Kafka Streams Application
    • Upgrading your Streams application from 0.11.0 to 1.0.0 does not require a broker upgrade. - A Kafka Streams 1.0.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
      • + A Kafka Streams 1.0.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though). + However, Kafka Streams 1.0 requires 0.10 message format or newer and does not work with older message formats.
    • If you are monitoring on streams metrics, you will need make some changes to the metrics names in your reporting and monitoring code, because the metrics sensor hierarchy was changed.
    • There are a few public APIs including ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs. We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade. From 4ee5182f35d612daa223f890b9eaad08655b761b Mon Sep 17 00:00:00 2001 From: Mats Julian Olsen Date: Wed, 7 Feb 2018 20:53:07 +0100 Subject: [PATCH 0018/1847] MINOR: Add attributes `processedKeys` and `processedValues` to MockProcessorSupplier (#3999) Author: Mats Julian Olsen Reviewers: Guozhang Wang , Matthias J. Sax --- .../java/org/apache/kafka/test/MockProcessorSupplier.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java b/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java index c464aaafbf332..bdc8d40bdab1f 100644 --- a/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java +++ b/streams/src/test/java/org/apache/kafka/test/MockProcessorSupplier.java @@ -31,6 +31,9 @@ public class MockProcessorSupplier implements ProcessorSupplier { public final ArrayList processed = new ArrayList<>(); + public final ArrayList processedKeys = new ArrayList<>(); + public final ArrayList processedValues = new ArrayList<>(); + public final ArrayList punctuatedStreamTime = new ArrayList<>(); public final ArrayList punctuatedSystemTime = new ArrayList<>(); @@ -86,6 +89,8 @@ public void punctuate(long timestamp) { @Override public void process(K key, V value) { + processedKeys.add(key); + processedValues.add(value); processed.add((key == null ? "null" : key) + ":" + (value == null ? "null" : value)); From 5d69a79948bce0fab4705c7d7d0f3b02f2548c0d Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 7 Feb 2018 14:08:53 -0800 Subject: [PATCH 0019/1847] KAFKA-4641: Add more unit test for stream thread (#4531) Before the patch, jacoco coverage test: Element Missed Instructions Cov. Missed Branches Cov. Missed Cxty Missed Lines Missed Methods Missed Classes Total 3,386 of 22,177 84% 336 of 1,639 79% 350 1,589 526 4,451 103 768 1 102 StreamThread 77% 76% 27 102 48 299 1 31 0 1 After the patch: Element Missed Instructions Cov. Missed Branches Cov. Missed Cxty Missed Lines Missed Methods Missed Classes Total 3,329 of 22,180 84% 329 of 1,639 79% 345 1,590 516 4,452 102 769 1 102 StreamThread 81% 80% 23 103 39 300 1 32 0 1 Reviewers: Bill Bejeck , Matthias J. Sax , Damian Guy --- .../processor/internals/StreamThread.java | 6 +- .../processor/internals/StreamThreadTest.java | 161 +++++++++++++++++- 2 files changed, 161 insertions(+), 6 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java index 064a293551588..5e25d02973aef 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java @@ -1183,8 +1183,12 @@ public String toString(final String indent) { return sb.toString(); } - // this is for testing only + // the following are for testing only TaskManager taskManager() { return taskManager; } + + Map>> standbyRecords() { + return standbyRecords; + } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index e67fe14503cb3..cc056044792b0 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.InvalidOffsetException; import org.apache.kafka.clients.consumer.MockConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.MockProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.common.Cluster; @@ -40,7 +41,13 @@ import org.apache.kafka.streams.kstream.internals.ConsumedInternal; import org.apache.kafka.streams.kstream.internals.InternalStreamsBuilder; import org.apache.kafka.streams.kstream.internals.InternalStreamsBuilderTest; +import org.apache.kafka.streams.kstream.internals.MaterializedInternal; import org.apache.kafka.streams.processor.LogAndSkipOnInvalidTimestamp; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; +import org.apache.kafka.streams.processor.PunctuationType; +import org.apache.kafka.streams.processor.Punctuator; import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.TaskMetadata; import org.apache.kafka.streams.processor.ThreadMetadata; @@ -100,13 +107,16 @@ public void setUp() { } private final String topic1 = "topic1"; + private final String topic2 = "topic2"; private final TopicPartition t1p1 = new TopicPartition(topic1, 1); private final TopicPartition t1p2 = new TopicPartition(topic1, 2); + private final TopicPartition t2p1 = new TopicPartition(topic2, 1); // task0 is unused private final TaskId task1 = new TaskId(0, 1); private final TaskId task2 = new TaskId(0, 2); + private final TaskId task3 = new TaskId(1, 1); private Properties configProps(final boolean enableEos) { return new Properties() { @@ -129,7 +139,7 @@ private Properties configProps(final boolean enableEos) { public void testPartitionAssignmentChangeForSingleGroup() { internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1); - final StreamThread thread = getStreamThread(); + final StreamThread thread = createStreamThread(clientId, config, false); final StateListenerStub stateListener = new StateListenerStub(); thread.setStateListener(stateListener); @@ -685,10 +695,6 @@ public void onChange(final Thread thread, final ThreadStateTransitionValidator n } } - private StreamThread getStreamThread() { - return createStreamThread(clientId, config, false); - } - @Test public void shouldReturnActiveTaskMetadataWhileRunningState() throws InterruptedException { internalTopologyBuilder.addSource(null, "source", null, null, null, topic1); @@ -759,6 +765,151 @@ public void shouldReturnStandbyTaskMetadataWhileRunningState() throws Interrupte assertTrue(threadMetadata.activeTasks().isEmpty()); } + @SuppressWarnings("unchecked") + @Test + public void shouldUpdateStandbyTask() { + final String storeName1 = "count-one"; + final String storeName2 = "table-two"; + final String changelogName = applicationId + "-" + storeName1 + "-changelog"; + final TopicPartition partition1 = new TopicPartition(changelogName, 1); + final TopicPartition partition2 = t2p1; + internalStreamsBuilder.stream(Collections.singleton(topic1), consumed) + .groupByKey().count(Materialized.>as(storeName1)); + internalStreamsBuilder.table(topic2, new ConsumedInternal(), new MaterializedInternal(Materialized.as(storeName2), internalStreamsBuilder, "")); + + final StreamThread thread = createStreamThread(clientId, config, false); + final MockConsumer restoreConsumer = clientSupplier.restoreConsumer; + restoreConsumer.updatePartitions(changelogName, + Collections.singletonList(new PartitionInfo(changelogName, + 1, + null, + new Node[0], + new Node[0]))); + + restoreConsumer.assign(Utils.mkSet(partition1, partition2)); + restoreConsumer.updateEndOffsets(Collections.singletonMap(partition1, 10L)); + restoreConsumer.updateBeginningOffsets(Collections.singletonMap(partition1, 0L)); + restoreConsumer.updateEndOffsets(Collections.singletonMap(partition2, 10L)); + restoreConsumer.updateBeginningOffsets(Collections.singletonMap(partition2, 0L)); + // let the store1 be restored from 0 to 10; store2 be restored from 0 to (committed offset) 5 + clientSupplier.consumer.assign(Utils.mkSet(partition2)); + clientSupplier.consumer.commitSync(Collections.singletonMap(partition2, new OffsetAndMetadata(5L, ""))); + + for (long i = 0L; i < 10L; i++) { + restoreConsumer.addRecord(new ConsumerRecord<>(changelogName, 1, i, ("K" + i).getBytes(), ("V" + i).getBytes())); + restoreConsumer.addRecord(new ConsumerRecord<>(topic2, 1, i, ("K" + i).getBytes(), ("V" + i).getBytes())); + } + + thread.setState(StreamThread.State.RUNNING); + + thread.rebalanceListener.onPartitionsRevoked(null); + + final Map> standbyTasks = new HashMap<>(); + + // assign single partition + standbyTasks.put(task1, Collections.singleton(t1p1)); + standbyTasks.put(task3, Collections.singleton(t2p1)); + + thread.taskManager().setAssignmentMetadata(Collections.>emptyMap(), standbyTasks); + + thread.rebalanceListener.onPartitionsAssigned(Collections.emptyList()); + + thread.runOnce(-1); + + final StandbyTask standbyTask1 = thread.taskManager().standbyTask(partition1); + final StandbyTask standbyTask2 = thread.taskManager().standbyTask(partition2); + final KeyValueStore store1 = (KeyValueStore) standbyTask1.getStore(storeName1); + final KeyValueStore store2 = (KeyValueStore) standbyTask2.getStore(storeName2); + + assertEquals(10L, store1.approximateNumEntries()); + assertEquals(5L, store2.approximateNumEntries()); + assertEquals(Collections.singleton(partition2), restoreConsumer.paused()); + assertEquals(1, thread.standbyRecords().size()); + assertEquals(5, thread.standbyRecords().get(partition2).size()); + } + + @Test + public void shouldPunctuateActiveTask() { + final List punctuatedStreamTime = new ArrayList<>(); + final List punctuatedWallClockTime = new ArrayList<>(); + final ProcessorSupplier punctuateProcessor = new ProcessorSupplier() { + @Override + public Processor get() { + return new Processor() { + @Override + public void init(ProcessorContext context) { + context.schedule(100L, PunctuationType.STREAM_TIME, new Punctuator() { + @Override + public void punctuate(long timestamp) { + punctuatedStreamTime.add(timestamp); + } + }); + context.schedule(100L, PunctuationType.WALL_CLOCK_TIME, new Punctuator() { + @Override + public void punctuate(long timestamp) { + punctuatedWallClockTime.add(timestamp); + } + }); + } + + @Override + public void process(Object key, Object value) { } + + @SuppressWarnings("deprecation") + @Override + public void punctuate(long timestamp) { } + + @Override + public void close() { } + }; + } + }; + + internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).process(punctuateProcessor); + + final StreamThread thread = createStreamThread(clientId, config, false); + + thread.setState(StreamThread.State.RUNNING); + + thread.rebalanceListener.onPartitionsRevoked(null); + final List assignedPartitions = new ArrayList<>(); + + final Map> activeTasks = new HashMap<>(); + + // assign single partition + assignedPartitions.add(t1p1); + activeTasks.put(task1, Collections.singleton(t1p1)); + + thread.taskManager().setAssignmentMetadata(activeTasks, Collections.>emptyMap()); + + thread.rebalanceListener.onPartitionsAssigned(assignedPartitions); + clientSupplier.consumer.assign(assignedPartitions); + clientSupplier.consumer.updateBeginningOffsets(Collections.singletonMap(t1p1, 0L)); + + thread.runOnce(-1); + + assertEquals(0, punctuatedStreamTime.size()); + assertEquals(0, punctuatedWallClockTime.size()); + + mockTime.sleep(100L); + for (long i = 0L; i < 10L; i++) { + clientSupplier.consumer.addRecord(new ConsumerRecord<>(topic1, 1, i, i * 100L, TimestampType.CREATE_TIME, ConsumerRecord.NULL_CHECKSUM, ("K" + i).getBytes().length, ("V" + i).getBytes().length, ("K" + i).getBytes(), ("V" + i).getBytes())); + } + + thread.runOnce(-1); + + assertEquals(1, punctuatedStreamTime.size()); + assertEquals(1, punctuatedWallClockTime.size()); + + mockTime.sleep(100L); + + thread.runOnce(-1); + + // we should skip stream time punctuation, only trigger wall-clock time punctuation + assertEquals(1, punctuatedStreamTime.size()); + assertEquals(2, punctuatedWallClockTime.size()); + } + @Test public void shouldAlwaysUpdateTasksMetadataAfterChangingState() throws InterruptedException { final StreamThread thread = createStreamThread(clientId, config, false); From 68ec1a3cce22706faf762de7d2d97b490c47fd90 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Wed, 7 Feb 2018 16:20:18 -0800 Subject: [PATCH 0020/1847] KAFKA-6519; Reduce log level for normal replica fetch errors (#4501) Out of range and not leader errors are common in replica fetchers and not necessarily an indication of a problem. This patch therefore reduces the log level for log messages corresponding to these errors from `ERROR` to `INFO`. Additionally, this patch removes some redundant information in the log message which is already present in the log context. Reviewers: Ismael Juma --- .../consumer/ConsumerFetcherManager.scala | 4 +-- .../consumer/ConsumerFetcherThread.scala | 8 +++-- .../kafka/server/AbstractFetcherThread.scala | 33 +++++++++++-------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala b/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala index 23f53569a6b2a..e84472f06bb96 100755 --- a/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala +++ b/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala @@ -114,9 +114,7 @@ class ConsumerFetcherManager(private val consumerIdString: String, } override def createFetcherThread(fetcherId: Int, sourceBroker: BrokerEndPoint): AbstractFetcherThread = { - new ConsumerFetcherThread( - "ConsumerFetcherThread-%s-%d-%d".format(consumerIdString, fetcherId, sourceBroker.id), - config, sourceBroker, partitionMap, this) + new ConsumerFetcherThread(consumerIdString, fetcherId, config, sourceBroker, partitionMap, this) } def startConnections(topicInfos: Iterable[PartitionTopicInfo], cluster: Cluster) { diff --git a/core/src/main/scala/kafka/consumer/ConsumerFetcherThread.scala b/core/src/main/scala/kafka/consumer/ConsumerFetcherThread.scala index 705dc249bf302..ac83fa17a767b 100644 --- a/core/src/main/scala/kafka/consumer/ConsumerFetcherThread.scala +++ b/core/src/main/scala/kafka/consumer/ConsumerFetcherThread.scala @@ -34,12 +34,13 @@ import org.apache.kafka.common.requests.EpochEndOffset @deprecated("This class has been deprecated and will be removed in a future release. " + "Please use org.apache.kafka.clients.consumer.internals.Fetcher instead.", "0.11.0.0") -class ConsumerFetcherThread(name: String, +class ConsumerFetcherThread(consumerIdString: String, + fetcherId: Int, val config: ConsumerConfig, sourceBroker: BrokerEndPoint, partitionMap: Map[TopicPartition, PartitionTopicInfo], val consumerFetcherManager: ConsumerFetcherManager) - extends AbstractFetcherThread(name = name, + extends AbstractFetcherThread(name = s"ConsumerFetcherThread-$consumerIdString-$fetcherId-${sourceBroker.id}", clientId = config.clientId, sourceBroker = sourceBroker, fetchBackOffMs = config.refreshLeaderBackoffMs, @@ -49,6 +50,9 @@ class ConsumerFetcherThread(name: String, type REQ = FetchRequest type PD = PartitionData + this.logIdent = s"[ConsumerFetcher consumerId=$consumerIdString, leaderId=${sourceBroker.id}, " + + s"fetcherId=$fetcherId] " + private val clientId = config.clientId private val fetchSize = config.fetchMessageMaxBytes diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala index 39a70321a6ef0..8d787c96da6e3 100755 --- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala @@ -47,8 +47,7 @@ abstract class AbstractFetcherThread(name: String, val sourceBroker: BrokerEndPoint, fetchBackOffMs: Int = 0, isInterruptible: Boolean = true, - includeLogTruncation: Boolean - ) + includeLogTruncation: Boolean) extends ShutdownableThread(name, isInterruptible) { type REQ <: FetchRequest @@ -140,16 +139,15 @@ abstract class AbstractFetcherThread(name: String, private def processFetchRequest(fetchRequest: REQ) { val partitionsWithError = mutable.Set[TopicPartition]() - var responseData: Seq[(TopicPartition, PD)] = Seq.empty try { - trace(s"Issuing fetch to broker ${sourceBroker.id}, request: $fetchRequest") + trace(s"Sending fetch request $fetchRequest") responseData = fetch(fetchRequest) } catch { case t: Throwable => if (isRunning) { - warn(s"Error in fetch to broker ${sourceBroker.id}, request $fetchRequest", t) + warn(s"Error in response for fetch request $fetchRequest", t) inLock(partitionMapLock) { partitionsWithError ++= partitionStates.partitionSet.asScala // there is an error occurred while fetching partitions, sleep a while @@ -210,27 +208,34 @@ abstract class AbstractFetcherThread(name: String, try { val newOffset = handleOffsetOutOfRange(topicPartition) partitionStates.updateAndMoveToEnd(topicPartition, new PartitionFetchState(newOffset)) - error(s"Current offset ${currentPartitionFetchState.fetchOffset} for partition $topicPartition out of range; reset offset to $newOffset") + info(s"Current offset ${currentPartitionFetchState.fetchOffset} for partition $topicPartition is " + + s"out of range, which typically implies a leader change. Reset fetch offset to $newOffset") } catch { case e: FatalExitError => throw e case e: Throwable => - error(s"Error getting offset for partition $topicPartition from broker ${sourceBroker.id}", e) + error(s"Error getting offset for partition $topicPartition", e) partitionsWithError += topicPartition } + + case Errors.NOT_LEADER_FOR_PARTITION => + info(s"Remote broker is not the leader for partition $topicPartition, which could indicate " + + "that the partition is being moved") + partitionsWithError += topicPartition + case _ => - if (isRunning) { - error(s"Error for partition $topicPartition from broker ${sourceBroker.id}", partitionData.exception.get) - partitionsWithError += topicPartition - } + error(s"Error for partition $topicPartition at offset ${currentPartitionFetchState.fetchOffset}", + partitionData.exception.get) + partitionsWithError += topicPartition } }) } } } - if (partitionsWithError.nonEmpty) - debug(s"handling partitions with error for $partitionsWithError") - handlePartitionsWithErrors(partitionsWithError) + if (partitionsWithError.nonEmpty) { + debug(s"Handling errors for partitions $partitionsWithError") + handlePartitionsWithErrors(partitionsWithError) + } } def markPartitionsForTruncation(topicPartition: TopicPartition, truncationOffset: Long) { From 67803384d9b7959661bb2a32129b03a374507c8a Mon Sep 17 00:00:00 2001 From: Bill Bejeck Date: Wed, 7 Feb 2018 20:21:35 -0500 Subject: [PATCH 0021/1847] MINOR: adding system tests for how streams functions with broker faiures (#4513) System test for two cases: * Starting a multi-node streams application with the broker down initially, broker starts and confirm rebalance completes and streams application still able to process records. * Multi-node streams app running, broker goes down, stop stream instance(s) confirm after broker comes back remaining streams instance(s) still function. Reviewers: Guozhang Wang , Matthias J. Sax --- .../StreamsBrokerDownResilienceTest.java | 10 +- tests/kafkatest/services/streams.py | 18 +++ .../streams_broker_down_resilience_test.py | 138 +++++++++++++++--- 3 files changed, 143 insertions(+), 23 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java index c8462ca40e80a..ed4cd276b78b3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsBrokerDownResilienceTest.java @@ -78,13 +78,16 @@ public static void main(String[] args) { System.exit(1); } - final StreamsBuilder builder = new StreamsBuilder(); builder.stream(Collections.singletonList(SOURCE_TOPIC_1), Consumed.with(stringSerde, stringSerde)) .peek(new ForeachAction() { + int messagesProcessed = 0; @Override public void apply(String key, String value) { System.out.println("received key " + key + " and value " + value); + messagesProcessed++; + System.out.println("processed" + messagesProcessed + "messages"); + System.out.flush(); } }).to(SINK_TOPIC); @@ -104,8 +107,9 @@ public void uncaughtException(final Thread t, final Throwable e) { Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { - System.out.println("Shutting down streams now"); - streams.close(10, TimeUnit.SECONDS); + streams.close(30, TimeUnit.SECONDS); + System.out.println("Complete shutdown of streams resilience test app now"); + System.out.flush(); } })); diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py index 9c4bd87dc52ae..0f484b4ca3384 100644 --- a/tests/kafkatest/services/streams.py +++ b/tests/kafkatest/services/streams.py @@ -208,9 +208,27 @@ def __init__(self, test_context, kafka, eosEnabled): "org.apache.kafka.streams.tests.BrokerCompatibilityTest", eosEnabled) + class StreamsBrokerDownResilienceService(StreamsTestBaseService): def __init__(self, test_context, kafka, configs): super(StreamsBrokerDownResilienceService, self).__init__(test_context, kafka, "org.apache.kafka.streams.tests.StreamsBrokerDownResilienceTest", configs) + + def start_cmd(self, node): + args = self.args.copy() + args['kafka'] = self.kafka.bootstrap_servers(validate=False) + args['state_dir'] = self.PERSISTENT_ROOT + args['stdout'] = self.STDOUT_FILE + args['stderr'] = self.STDERR_FILE + args['pidfile'] = self.PID_FILE + args['log4j'] = self.LOG4J_CONFIG_FILE + args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) + + cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + "INCLUDE_TEST_JARS=true %(kafka_run_class)s %(streams_class_name)s " \ + " %(kafka)s %(state_dir)s %(user_test_args)s %(user_test_args1)s %(user_test_args2)s" \ + " %(user_test_args3)s & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args + + return cmd diff --git a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py index bd90d9f1a2ae9..7a0560d0e996e 100644 --- a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py +++ b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py @@ -40,61 +40,76 @@ def __init__(self, test_context): num_nodes=1, zk=self.zk, topics={ - self.inputTopic: {'partitions': 1, 'replication-factor': 1}, + self.inputTopic: {'partitions': 3, 'replication-factor': 1}, self.outputTopic: {'partitions': 1, 'replication-factor': 1} }) - def get_consumer(self): + def get_consumer(self, num_messages): return VerifiableConsumer(self.test_context, 1, self.kafka, self.outputTopic, "stream-broker-resilience-verify-consumer", - max_messages=self.num_messages) + max_messages=num_messages) - def get_producer(self): + def get_producer(self, num_messages): return VerifiableProducer(self.test_context, 1, self.kafka, self.inputTopic, - max_messages=self.num_messages, + max_messages=num_messages, acks=1) - def assert_produce_consume(self, test_state): - producer = self.get_producer() + def assert_produce_consume(self, test_state, num_messages=5): + producer = self.get_producer(num_messages) producer.start() - wait_until(lambda: producer.num_acked > 0, + wait_until(lambda: producer.num_acked >= num_messages, timeout_sec=30, err_msg="At %s failed to send messages " % test_state) - consumer = self.get_consumer() + consumer = self.get_consumer(num_messages) consumer.start() - wait_until(lambda: consumer.total_consumed() > 0, + wait_until(lambda: consumer.total_consumed() >= num_messages, timeout_sec=60, err_msg="At %s streams did not process messages in 60 seconds " % test_state) - def setUp(self): - self.zk.start() - - def test_streams_resilient_to_broker_down(self): - self.kafka.start() - + @staticmethod + def get_configs(extra_configs=""): # Consumer max.poll.interval > min(max.block.ms, ((retries + 1) * request.timeout) consumer_poll_ms = "consumer.max.poll.interval.ms=50000" retries_config = "producer.retries=2" request_timeout = "producer.request.timeout.ms=15000" max_block_ms = "producer.max.block.ms=30000" + # java code expects configs in key=value,key=value format + updated_configs = consumer_poll_ms + "," + retries_config + "," + request_timeout + "," + max_block_ms + extra_configs + + return updated_configs + + def wait_for_verification(self, processor, message, file, num_lines=1): + wait_until(lambda: self.verify_from_file(processor, message, file) >= num_lines, + timeout_sec=60, + err_msg="Did expect to read '%s' from %s" % (message, processor.node.account)) + + @staticmethod + def verify_from_file(processor, message, file): + result = processor.node.account.ssh_output("grep '%s' %s | wc -l" % (message, file), allow_fail=False) + return int(result) + + + def setUp(self): + self.zk.start() + + def test_streams_resilient_to_broker_down(self): + self.kafka.start() + # Broker should be down over 2x of retries * timeout ms # So with (2 * 15000) = 30 seconds, we'll set downtime to 70 seconds broker_down_time_in_seconds = 70 - # java code expects configs in key=value,key=value format - updated_configs = consumer_poll_ms + "," + retries_config + "," + request_timeout + "," + max_block_ms - - processor = StreamsBrokerDownResilienceService(self.test_context, self.kafka, updated_configs) + processor = StreamsBrokerDownResilienceService(self.test_context, self.kafka, self.get_configs()) processor.start() # until KIP-91 is merged we'll only send 5 messages to assert Kafka Streams is running before taking the broker down @@ -112,3 +127,86 @@ def test_streams_resilient_to_broker_down(self): self.assert_produce_consume("after_broker_stop") self.kafka.stop() + + def test_streams_runs_with_broker_down_initially(self): + self.kafka.start() + node = self.kafka.leader(self.inputTopic) + self.kafka.stop_node(node) + + configs = self.get_configs(extra_configs=",application.id=starting_wo_broker_id") + + # start streams with broker down initially + processor = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) + processor.start() + + processor_2 = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) + processor_2.start() + + processor_3 = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) + processor_3.start() + + broker_unavailable_message = "Broker may not be available" + + # verify streams instances unable to connect to broker, kept trying + self.wait_for_verification(processor, broker_unavailable_message, processor.LOG_FILE, 100) + self.wait_for_verification(processor_2, broker_unavailable_message, processor_2.LOG_FILE, 100) + self.wait_for_verification(processor_3, broker_unavailable_message, processor_3.LOG_FILE, 100) + + # now start broker + self.kafka.start_node(node) + + # assert streams can process when starting with broker down + self.assert_produce_consume("running_with_broker_down_initially", num_messages=9) + + message = "processed3messages" + # need to show all 3 instances processed messages + self.wait_for_verification(processor, message, processor.STDOUT_FILE) + self.wait_for_verification(processor_2, message, processor_2.STDOUT_FILE) + self.wait_for_verification(processor_3, message, processor_3.STDOUT_FILE) + + self.kafka.stop() + + def test_streams_should_scale_in_while_brokers_down(self): + self.kafka.start() + + configs = self.get_configs(extra_configs=",application.id=shutdown_with_broker_down") + + processor = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) + processor.start() + + processor_2 = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) + processor_2.start() + + processor_3 = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) + processor_3.start() + + # need to wait for rebalance once + self.wait_for_verification(processor_3, "State transition from REBALANCING to RUNNING", processor_3.LOG_FILE) + + # assert streams can process when starting with broker down + self.assert_produce_consume("waiting for rebalance to complete", num_messages=9) + + message = "processed3messages" + + self.wait_for_verification(processor, message, processor.STDOUT_FILE) + self.wait_for_verification(processor_2, message, processor_2.STDOUT_FILE) + self.wait_for_verification(processor_3, message, processor_3.STDOUT_FILE) + + node = self.kafka.leader(self.inputTopic) + self.kafka.stop_node(node) + + processor.stop() + processor_2.stop() + + shutdown_message = "Complete shutdown of streams resilience test app now" + self.wait_for_verification(processor, shutdown_message, processor.STDOUT_FILE) + self.wait_for_verification(processor_2, shutdown_message, processor_2.STDOUT_FILE) + + self.kafka.start_node(node) + + self.assert_produce_consume("sending_message_after_stopping_streams_instance_bouncing_broker", num_messages=9) + + self.wait_for_verification(processor_3, "processed9messages", processor_3.STDOUT_FILE) + + self.kafka.stop() + From e7905fc7c3398490e02cddb3bcf30590b2f6d7b7 Mon Sep 17 00:00:00 2001 From: Dmitry Minkovsky Date: Wed, 7 Feb 2018 20:41:39 -0500 Subject: [PATCH 0022/1847] MINOR: Update TupleForwarder comment (#4414) Reviewers: Guozhang Wang --- .../apache/kafka/streams/kstream/internals/TupleForwarder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java index 4c02d1d3fe969..ff3ef44894be6 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java @@ -23,7 +23,8 @@ /** * This class is used to determine if a processor should forward values to child nodes. - * Forwarding only occurs when caching is not enabled. + * Forwarding by this class only occurs when caching is not enabled. If caching is enabled, + * forwarding occurs in the flush listener when the cached store flushes. * * @param * @param From 8a2d00fc44eb48bf80fd2383178665986a73cdd8 Mon Sep 17 00:00:00 2001 From: Guangxian Date: Thu, 8 Feb 2018 09:44:40 +0800 Subject: [PATCH 0023/1847] KAFKA-6405: Fix incorrect comment in MetadataUpdater (#4361) * Fix incorrect comment in MetadataUpdater * Fix comment for handleCompletedMetadataResponse --- .../main/java/org/apache/kafka/clients/MetadataUpdater.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java b/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java index cb821d6a86064..126728342d4c7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java +++ b/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java @@ -54,7 +54,7 @@ interface MetadataUpdater { long maybeUpdate(long now); /** - * If `request` is a metadata request, handles it and return `true`. Otherwise, returns `false`. + * Handle disconnections for metadata requests. * * This provides a mechanism for the `MetadataUpdater` implementation to use the NetworkClient instance for its own * requests with special handling for disconnections of such requests. @@ -70,7 +70,7 @@ interface MetadataUpdater { void handleAuthenticationFailure(AuthenticationException exception); /** - * If `request` is a metadata request, handles it and returns `true`. Otherwise, returns `false`. + * Handle responses for metadata requests. * * This provides a mechanism for the `MetadataUpdater` implementation to use the NetworkClient instance for its own * requests with special handling for completed receives of such requests. From 79f22805a75cf2eb49560bd8387cb6ec14769bd7 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 8 Feb 2018 09:30:25 -0800 Subject: [PATCH 0024/1847] MINOR: Update jetty, jackson, gradle and jacoco (#4547) * MINOR: Update gradle, jackson and jacoco - Gradle update adds support for Java 10 - Jacoco update adds support for Java 9 - Jackson bug fix update adds more serialization robustness checks * Update Jetty Reviewers: Rajini Sivaram --- build.gradle | 9 +++++++-- gradle/dependencies.gradle | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 430d9896dcf4c..5e4c35643c2a1 100644 --- a/build.gradle +++ b/build.gradle @@ -79,7 +79,7 @@ allprojects { } ext { - gradleVersion = "4.4.1" + gradleVersion = "4.5.1" buildVersionFileName = "kafka-version.properties" maxPermSizeArgs = [] @@ -384,7 +384,12 @@ subprojects { // Ignore core since its a scala project if (it.path != ':core') { - // NOTE: Gradles Jacoco plugin does not support "offline instrumentation" this means that classes mocked by PowerMock + + jacoco { + toolVersion = "0.8.0" + } + + // NOTE: Jacoco Gradle plugin does not support "offline instrumentation" this means that classes mocked by PowerMock // may report 0 coverage, since the source was modified after initial instrumentation. // See https://github.com/jacoco/jacoco/issues/51 jacocoTestReport { diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 963fd031b3c8b..2ac496b4ca9ac 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -52,8 +52,8 @@ versions += [ argparse4j: "0.7.0", bcpkix: "1.58", easymock: "3.5.1", - jackson: "2.9.3", - jetty: "9.2.22.v20170606", + jackson: "2.9.4", + jetty: "9.2.24.v20180105", jersey: "2.25.1", jmh: "1.19", log4j: "1.2.17", From 077fd9ced3f5c7101c0d4e91c0ae3aeb05f9318f Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Thu, 8 Feb 2018 09:31:58 -0800 Subject: [PATCH 0025/1847] HOTFIX: Fix broken javadoc links on web docs(#4543) Reviewers: Matthias J. Sax --- .../developer-guide/app-reset-tool.html | 2 +- .../developer-guide/config-streams.html | 32 ++--- docs/streams/developer-guide/dsl-api.html | 110 +++++++++--------- .../developer-guide/interactive-queries.html | 6 +- .../developer-guide/processor-api.html | 4 +- docs/upgrade.html | 2 +- 6 files changed, 78 insertions(+), 78 deletions(-) diff --git a/docs/streams/developer-guide/app-reset-tool.html b/docs/streams/developer-guide/app-reset-tool.html index 769dc39b1e4de..84b6930f10a44 100644 --- a/docs/streams/developer-guide/app-reset-tool.html +++ b/docs/streams/developer-guide/app-reset-tool.html @@ -141,7 +141,7 @@

      Step 1: Run the application reset tool
    • The API method KafkaStreams#cleanUp() in your application code.
      • -
    • Manually delete the corresponding local state directory (default location: /var/lib/kafka-streams/<application.id>). For more information, see state.dir StreamsConfig class.
      • +
    • Manually delete the corresponding local state directory (default location: /var/lib/kafka-streams/<application.id>). For more information, see Streams javadocs.

    diff --git a/docs/streams/developer-guide/config-streams.html b/docs/streams/developer-guide/config-streams.html index 8fbb6d5c1632d..1f18a95367994 100644 --- a/docs/streams/developer-guide/config-streams.html +++ b/docs/streams/developer-guide/config-streams.html @@ -59,7 +59,7 @@

    Configuration parameter reference

    -

    This section contains the most common Streams configuration parameters. For a full reference, see the Streams and Client Javadocs.

    +

    This section contains the most common Streams configuration parameters. For a full reference, see the Streams Javadocs.

    -

    default.production.exception.handler

    +

    default.production.exception.handler

    The default production exception handler allows you to manage exceptions triggered when trying to interact with a broker - such as attempting to produce a record that is too large. By default, Kafka provides and uses the DefaultProductionExceptionHandler + such as attempting to produce a record that is too large. By default, Kafka provides and uses the DefaultProductionExceptionHandler that always fails when these exceptions occur.

    Each exception handler can return a FAIL or CONTINUE depending on the record and the exception thrown. Returning FAIL will signal that Streams should shut down and CONTINUE will signal that Streams @@ -399,7 +399,7 @@

    num.stream.threads

    partition.grouper

    A partition grouper creates a list of stream tasks from the partitions of source topics, where each created task is assigned with a group of source topic partitions. - The default implementation provided by Kafka Streams is DefaultPartitionGrouper. + The default implementation provided by Kafka Streams is DefaultPartitionGrouper. It assigns each task with one partition for each of the source topic partitions. The generated number of tasks equals the largest number of partitions among the input topics. Usually an application does not need to customize the partition grouper.

    @@ -426,10 +426,10 @@

    state.dir

    timestamp.extractor

    -

    A timestamp extractor pulls a timestamp from an instance of ConsumerRecord. +

    A timestamp extractor pulls a timestamp from an instance of ConsumerRecord. Timestamps are used to control the progress of streams.

    The default extractor is - FailOnInvalidTimestamp. + FailOnInvalidTimestamp. This extractor retrieves built-in timestamps that are automatically embedded into Kafka messages by the Kafka producer client since Kafka version 0.10. @@ -451,19 +451,19 @@

    state.dirIf you have data with invalid timestamps and want to process it, then there are two alternative extractors available. Both work on built-in timestamps, but handle invalid timestamps differently.

      -
    • LogAndSkipOnInvalidTimestamp: +
    • LogAndSkipOnInvalidTimestamp: This extractor logs a warn message and returns the invalid timestamp to Kafka Streams, which will not process but silently drop the record. This log-and-skip strategy allows Kafka Streams to make progress instead of failing if there are records with an invalid built-in timestamp in your input data.
      • -
    • UsePreviousTimeOnInvalidTimestamp. +
    • UsePreviousTimeOnInvalidTimestamp. This extractor returns the record’s built-in timestamp if it is valid (i.e. not negative). If the record does not have a valid built-in timestamps, the extractor returns the previously extracted valid timestamp from a record of the same topic partition as the current record as a timestamp estimation. In case that no timestamp can be estimated, it throws an exception.

    Another built-in extractor is - WallclockTimestampExtractor. + WallclockTimestampExtractor. This extractor does not actually “extract” a timestamp from the consumed record but rather returns the current time in milliseconds from the system clock (think: System.currentTimeMillis()), which effectively means Streams will operate on the basis of the so-called processing-time of events.

    @@ -515,9 +515,9 @@

    state.dir

    Kafka consumers and producer configuration parameters

    -

    You can specify parameters for the Kafka consumers and producers that are used internally. The consumer and producer settings +

    You can specify parameters for the Kafka consumers and producers that are used internally. The consumer and producer settings are defined by specifying parameters in a StreamsConfig instance.

    -

    In this example, the Kafka consumer session timeout is configured to be 60000 milliseconds in the Streams settings:

    +

    In this example, the Kafka consumer session timeout is configured to be 60000 milliseconds in the Streams settings:

    Properties streamsSettings = new Properties();
 // Example of a "normal" setting for Kafka Streams
 streamsSettings.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka-broker-01:9092");
@@ -603,7 +603,7 @@ 
    Default Values
    rocksdb.config.setter
    
           
    
             
    The RocksDB configuration. Kafka Streams uses RocksDB as the default storage engine for persistent stores. To change the default
-              configuration for RocksDB, implement RocksDBConfigSetter and provide your custom class via rocksdb.config.setter.
    
+              configuration for RocksDB, implement RocksDBConfigSetter and provide your custom class via rocksdb.config.setter.
    
               
    Here is an example that adjusts the memory size consumed by RocksDB.
    
               
        public static class CustomRocksDBConfig implements RocksDBConfigSetter {
 
@@ -704,7 +704,7 @@ 
    replication.factorNote
    
             
    A future version of Kafka Streams will allow developers to set their own app-specific configuration settings through
               StreamsConfig as well, which can then be accessed through
-              ProcessorContext.
    
+              ProcessorContext.
    
 
    
 
    
 
    
diff --git a/docs/streams/developer-guide/dsl-api.html b/docs/streams/developer-guide/dsl-api.html
index ba03e2e6f1bb3..f8fa8c9989d26 100644
--- a/docs/streams/developer-guide/dsl-api.html
+++ b/docs/streams/developer-guide/dsl-api.html
@@ -95,7 +95,7 @@ 
    OverviewKafka Streams Javadocs.
    
+            
    For a complete list of available API functionality, see also the Streams API docs.

    Creating source streams from Kafka

    @@ -119,7 +119,7 @@

    OverviewCreates a KStream from the specified Kafka input topics and interprets the data as a record stream. A KStream represents a partitioned record stream. - (details)

    + (details)

    In the case of a KStream, the local KStream instance of every application instance will be populated with data from only a subset of the partitions of the input topic. Collectively, across all application instances, all input topic partitions are read and processed.

    @@ -153,7 +153,7 @@

    OverviewReads the specified Kafka input topic into a KTable. The topic is interpreted as a changelog stream, where records with the same key are interpreted as UPSERT aka INSERT/UPDATE (when the record value is not null) or as DELETE (when the value is null) for that key. - (details)

    + (details)

    In the case of a KStream, the local KStream instance of every application instance will be populated with data from only a subset of the partitions of the input topic. Collectively, across all application instances, all input topic partitions are read and processed.

    @@ -178,7 +178,7 @@

    OverviewReads the specified Kafka input topic into a GlobalKTable. The topic is interpreted as a changelog stream, where records with the same key are interpreted as UPSERT aka INSERT/UPDATE (when the record value is not null) or as DELETE (when the value is null) for that key. - (details)

    + (details)

    In the case of a GlobalKTable, the local GlobalKTable instance of every application instance will be populated with data from only a subset of the partitions of the input topic. Collectively, across all application instances, all input topic partitions are read and processed.

    @@ -250,7 +250,7 @@

    OverviewBranch (or split) a KStream based on the supplied predicates into one or more KStream instances. - (details)

    + (details)

    Predicates are evaluated in order. A record is placed to one and only one output stream on the first match: if the n-th predicate evaluates to true, the record is placed to n-th stream. If no predicate matches, the the record is dropped.

    @@ -278,8 +278,8 @@

    OverviewEvaluates a boolean function for each element and retains those for which the function returns true. - (KStream details, - KTable details)

    + (KStream details, + KTable details)

    KStream<String, Long> stream = ...;
 
 // A filter that selects (keeps) only positive numbers
@@ -305,8 +305,8 @@ 
    OverviewEvaluates a boolean function for each element and drops those for which the function returns true.
-                            (KStream details,
-                            KTable details)
    
+                            (KStream details,
+                            KTable details)
    
                             
    KStream<String, Long> stream = ...;
 
 // An inverse filter that discards any negative numbers or zero
@@ -332,7 +332,7 @@ 
    OverviewTakes one record and produces zero, one, or more records.  You can modify the record keys and values, including
                             their types.
-                            (details)
    
+                            (details)
    
                             
    Marks the stream for data re-partitioning:
                                 Applying a grouping or a join after flatMap will result in re-partitioning of the records.
                                 If possible use flatMapValues instead, which will not cause data re-partitioning.
    
@@ -361,7 +361,7 @@ 
    OverviewTakes one record and produces zero, one, or more records, while retaining the key of the original record.
                             You can modify the record values and the value type.
-                            (details)
    
+                            (details)
    
                             
    flatMapValues is preferable to flatMap because it will not cause data re-partitioning.  However, you
                                 cannot modify the key or key type like flatMap does.
    
                             
    // Split a sentence into words.
@@ -381,7 +381,7 @@ 
    OverviewTerminal operation.  Performs a stateless action on each record.
-                            (details)
    
+                            (details)
    
                             
    You would use foreach to cause side effects based on the input data (similar to peek) and then stop
                                 further processing of the input data (unlike peek, which is not a terminal operation).
    
                             
    Note on processing guarantees: Any side effects of an action (such as writing to external systems) are not
@@ -410,7 +410,7 @@ 
    OverviewGroups the records by the existing key.
-                            (details)
    
+                            (details)
    
                             
    Grouping is a prerequisite for aggregating a stream or a table
                                 and ensures that data is properly partitioned (“keyed”) for subsequent operations.
    
                             
    When to set explicit SerDes:
@@ -455,8 +455,8 @@ 
    OverviewGroups the records by a new key, which may be of a different key type.
                             When grouping a table, you may also specify a new value and value type.
                             groupBy is a shorthand for selectKey(...).groupByKey().
-                            (KStream details,
-                            KTable details)
    
+                            (KStream details,
+                            KTable details)
    
                             
    Grouping is a prerequisite for aggregating a stream or a table
                                 and ensures that data is properly partitioned (“keyed”) for subsequent operations.
    
                             
    When to set explicit SerDes:
@@ -532,7 +532,7 @@ 
    OverviewTakes one record and produces one record.  You can modify the record key and value, including their types.
-                            (details)
    
+                            (details)
    
                             
    Marks the stream for data re-partitioning:
                                 Applying a grouping or a join after map will result in re-partitioning of the records.
                                 If possible use mapValues instead, which will not cause data re-partitioning.
    
@@ -564,8 +564,8 @@ 
    OverviewTakes one record and produces one record, while retaining the key of the original record.
                             You can modify the record value and the value type.
-                            (KStream details,
-                            KTable details)
    
+                            (KStream details,
+                            KTable details)
    
                             
    mapValues is preferable to map because it will not cause data re-partitioning.  However, it does not
                                 allow you to modify the key or key type like map does.
    
                             
    KStream<byte[], String> stream = ...;
@@ -591,7 +591,7 @@ 
    OverviewPerforms a stateless action on each record, and returns an unchanged stream.
-                            (details)
    
+                            (details)
    
                             
    You would use peek to cause side effects based on the input data (similar to foreach) and continue
                                 processing the input data (unlike foreach, which is a terminal operation).  peek returns the input
                                 stream as-is;  if you need to modify the input stream, use map or mapValues instead.
    
@@ -623,7 +623,7 @@ 
    OverviewTerminal operation.  Prints the records to System.out.  See Javadocs for serde and toString()
                             caveats.
-                            (details)
    
+                            (details)
    
                             
    Calling print() is the same as calling foreach((key, value) -> System.out.println(key + ", " + value))
    
                             
    KStream<byte[], String> stream = ...;
 // print to sysout
@@ -641,7 +641,7 @@ 
    OverviewAssigns a new key – possibly of a new key type – to each record.
-                            (details)
    
+                            (details)
    
                             
    Calling selectKey(mapper) is the same as calling map((key, value) -> mapper(key, value), value).
    
                             
    Marks the stream for data re-partitioning:
                                 Applying a grouping or a join after selectKey will result in re-partitioning of the records.
    
@@ -669,7 +669,7 @@ 
    OverviewGet the changelog stream of this table.
-                            (details)
    
+                            (details)
    
                             
    KTable<byte[], String> table = ...;
 
 // Also, a variant of `toStream` exists that allows you
@@ -773,8 +773,8 @@ 
    OverviewRolling aggregation. Aggregates the values of (non-windowed) records by the grouped key.
                                 Aggregating is a generalization of reduce and allows, for example, the aggregate value to have a different
                                 type than the input values.
-                                (KGroupedStream details,
-                                KGroupedTable details)
    
+                                (KGroupedStream details,
+                                KGroupedTable details)
    
                                 
    When aggregating a grouped stream, you must provide an initializer (e.g., aggValue = 0) and an “adder”
                                     aggregator (e.g., aggValue + curValue).  When aggregating a grouped table, you must provide a
                                     “subtractor” aggregator (think: aggValue - oldValue).
    
@@ -876,8 +876,8 @@ 
    Overviewper window, by the grouped key.
                                 Aggregating is a generalization of reduce and allows, for example, the aggregate value to have a different
                                 type than the input values.
-                                (TimeWindowedKStream details,
-                                SessionWindowedKStream details)
    
+                                (TimeWindowedKStream details,
+                                SessionWindowedKStream details)
    
                                 
    You must provide an initializer (e.g.,  aggValue = 0), “adder” aggregator (e.g.,  aggValue + curValue),
                                     and a window.  When windowing based on sessions, you must additionally provide a “session merger” aggregator
                                     (e.g.,  mergedAggValue = leftAggValue + rightAggValue).
    
@@ -971,8 +971,8 @@ 
    OverviewRolling aggregation. Counts the number of records by the grouped key.
-                                (KGroupedStream details,
-                                KGroupedTable details)
    
+                                (KGroupedStream details,
+                                KGroupedTable details)
    
                                 
    Several variants of count exist, see Javadocs for details.
    
                                 
    KGroupedStream<String, Long> groupedStream = ...;
 KGroupedTable<String, Long> groupedTable = ...;
@@ -1002,8 +1002,8 @@ 
    OverviewWindowed aggregation.
                                 Counts the number of records, per window, by the grouped key.
-                                (TimeWindowedKStream details,
-                                SessionWindowedKStream details)
    
+                                (TimeWindowedKStream details,
+                                SessionWindowedKStream details)
    
                                 
    The windowed count turns a TimeWindowedKStream<K, V> or SessionWindowedKStream<K, V>
                                     into a windowed KTable<Windowed<K>, V>.
    
                                 
    Several variants of count exist, see Javadocs for details.
    
@@ -1036,8 +1036,8 @@ 
    OverviewRolling aggregation. Combines the values of (non-windowed) records by the grouped key.
                                 The current record value is combined with the last reduced value, and a new reduced value is returned.
                                 The result value type cannot be changed, unlike aggregate.
-                                (KGroupedStream details,
-                                KGroupedTable details)
    
+                                (KGroupedStream details,
+                                KGroupedTable details)
    
                                 
    When reducing a grouped stream, you must provide an “adder” reducer (e.g.,  aggValue + curValue).
                                     When reducing a grouped table, you must additionally provide a “subtractor” reducer (e.g.,
                                     aggValue - oldValue).
    
@@ -1120,8 +1120,8 @@ 
    Overviewnull key or value are ignored.
                                 The result value type cannot be changed, unlike aggregate.
-                                (TimeWindowedKStream details,
-                                SessionWindowedKStream details)
    
+                                (TimeWindowedKStream details,
+                                SessionWindowedKStream details)
    
                                 
    The windowed reduce turns a turns a TimeWindowedKStream<K, V> or a SessionWindowedKStream<K, V>
                                     into a windowed KTable<Windowed<K>, V>.
    
                                 
    Several variants of reduce exist, see Javadocs for details.
    
@@ -1646,7 +1646,7 @@ 
    OverviewPerforms an INNER JOIN of this stream with another stream.
                                     Even though this operation is windowed, the joined stream will be of type KStream<K, ...> rather than KStream<Windowed<K>, ...>.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    Causes data re-partitioning of a stream if and only if the stream was marked for re-partitioning (if both are marked, both are re-partitioned).
    
                                     
    Several variants of join exists, see the Javadocs for details.
    
@@ -1705,7 +1705,7 @@ 
    OverviewPerforms a LEFT JOIN of this stream with another stream.
                                     Even though this operation is windowed, the joined stream will be of type KStream<K, ...> rather than KStream<Windowed<K>, ...>.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    Causes data re-partitioning of a stream if and only if the stream was marked for re-partitioning (if both are marked, both are re-partitioned).
    
                                     
    Several variants of leftJoin exists, see the Javadocs for details.
    
@@ -1767,7 +1767,7 @@ 
    OverviewPerforms an OUTER JOIN of this stream with another stream.
                                     Even though this operation is windowed, the joined stream will be of type KStream<K, ...> rather than KStream<Windowed<K>, ...>.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    Causes data re-partitioning of a stream if and only if the stream was marked for re-partitioning (if both are marked, both are re-partitioned).
    
                                     
    Several variants of outerJoin exists, see the Javadocs for details.
    
@@ -1828,7 +1828,7 @@ 
    OverviewValueJoiner for the join, leftJoin, and
+                            ValueJoiner for the join, leftJoin, and
                             outerJoin methods, respectively, whenever a new input record is received on either side of the join.  An empty table
                             cell denotes that the ValueJoiner is not called at all.
    
                         
@@ -1994,7 +1994,7 @@ 
    OverviewPerforms an INNER JOIN of this table with another table.
                                     The result is an ever-updating KTable that represents the “current” result of the join.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    KTable<String, Long> left = ...;
 KTable<String, Double> right = ...;
@@ -2040,7 +2040,7 @@ 
    OverviewPerforms a LEFT JOIN of this table with another table.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    KTable<String, Long> left = ...;
 KTable<String, Double> right = ...;
@@ -2089,7 +2089,7 @@ 
    OverviewPerforms an OUTER JOIN of this table with another table.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    KTable<String, Long> left = ...;
 KTable<String, Double> right = ...;
@@ -2138,7 +2138,7 @@ 
    OverviewValueJoiner for the join, leftJoin, and
+                            ValueJoiner for the join, leftJoin, and
                             outerJoin methods, respectively, whenever a new input record is received on either side of the join.  An empty table
                             cell denotes that the ValueJoiner is not called at all.
    
                         
    
@@ -2302,7 +2302,7 @@ 
    OverviewPerforms an INNER JOIN of this stream with the table, effectively doing a table lookup.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    Causes data re-partitioning of the stream if and only if the stream was marked for re-partitioning.
    
                                     
    Several variants of join exists, see the Javadocs for details.
    
@@ -2355,7 +2355,7 @@ 
    OverviewPerforms a LEFT JOIN of this stream with the table, effectively doing a table lookup.
-                                    (details)
    
+                                    (details)
    
                                     
    Data must be co-partitioned: The input data for both sides must be co-partitioned.
    
                                     
    Causes data re-partitioning of the stream if and only if the stream was marked for re-partitioning.
    
                                     
    Several variants of leftJoin exists, see the Javadocs for details.
    
@@ -2411,7 +2411,7 @@ 
    OverviewValueJoiner for the join and leftJoin
+                            ValueJoiner for the join and leftJoin
                             methods, respectively, whenever a new input record is received on either side of the join.  An empty table
                             cell denotes that the ValueJoiner is not called at all.
    
                         
    
@@ -2573,7 +2573,7 @@ 
    OverviewPerforms an INNER JOIN of this stream with the global table, effectively doing a table lookup.
-                                    (details)
    
+                                    (details)
    
                                     
    The GlobalKTable is fully bootstrapped upon (re)start of a KafkaStreams instance, which means the table is fully populated with all the data in the underlying topic that is
                                         available at the time of the startup.  The actual data processing begins only once the bootstrapping has completed.
    
                                     
    Causes data re-partitioning of the stream if and only if the stream was marked for re-partitioning.
    
@@ -2627,7 +2627,7 @@ 
    OverviewPerforms a LEFT JOIN of this stream with the global table, effectively doing a table lookup.
-                                    (details)
    
+                                    (details)
    
                                     
    The GlobalKTable is fully bootstrapped upon (re)start of a KafkaStreams instance, which means the table is fully populated with all the data in the underlying topic that is
                                         available at the time of the startup.  The actual data processing begins only once the bootstrapping has completed.
    
                                     
    Causes data re-partitioning of the stream if and only if the stream was marked for re-partitioning.
    
@@ -2903,11 +2903,11 @@ 
    OverviewTerminal operation.  Applies a Processor to each record.
                             process() allows you to leverage the Processor API from the DSL.
-                            (details)
    
+                            (details)
    
                             
    This is essentially equivalent to adding the Processor via Topology#addProcessor() to your
                                 processor topology.
    
                             
    An example is available in the
-                                javadocs.
    
+                                javadocs.
    
                         
                     
                     
    
@@ -2938,14 +2938,14 @@ 
    OverviewApplies a ValueTransformer to each record, while retaining the key of the original record.
                             transformValues() allows you to leverage the Processor API from the DSL.
-                            (details)
    
+                            (details)
    
                             
    Each input record is transformed into exactly one output record (zero output records or multiple output records are not possible).
                                 The ValueTransformer may return null as the new value for a record.
    
                             
    transformValues is preferable to transform because it will not cause data re-partitioning.
    
                             
    transformValues is essentially equivalent to adding the ValueTransformer via Topology#addProcessor() to your
                                 processor topology.
    
                             
    An example is available in the
-                                javadocs.
    
+                                javadocs.
    
                         
                     
                     
    
@@ -3051,7 +3051,7 @@ 
    OverviewTerminal operation.  Write the records to a Kafka topic.
-                        (KStream details)
    
+                        (KStream details)
    
                         
    When to provide serdes explicitly:
    
                         
    \n");
+    }
+
+    private void addColumnValue(StringBuilder builder, String value) {
+        builder.append("");
+    }
+
+    /**
+     * Converts this config into an HTML table that can be embedded into docs.
+     * If dynamicUpdateModes is non-empty, a "Dynamic Update Mode" column
+     * will be included n the table with the value of the update mode. Default
+     * mode is "read-only".
+     * @param dynamicUpdateModes Config name -> update mode mapping
+     */
+    public String toHtmlTable(Map dynamicUpdateModes) {
+        boolean hasUpdateModes = !dynamicUpdateModes.isEmpty();
         List configs = sortedConfigs();
         StringBuilder b = new StringBuilder();
         b.append("
    Transform
    
@@ -2917,7 +2917,7 @@ 
    OverviewApplies a Transformer to each record.
                             transform() allows you to leverage the Processor API from the DSL.
-                            (details)
    
+                            (details)
    
                             
    Each input record is transformed into zero, one, or more output records (similar to the stateless flatMap).
                                 The Transformer must return null for zero output.
                                 You can modify the record’s key and value, including their types.
    
@@ -2927,7 +2927,7 @@ 
    Overviewtransform is essentially equivalent to adding the Transformer via Topology#addProcessor() to your
                                 processor topology.
    
                             
    An example is available in the
-                                javadocs.
+                                javadocs.
                                
    
                         
    		
                     
    ");
+        builder.append(headerName);
+        builder.append("");
+        builder.append(value);
+        builder.append("
    \n");
         b.append("\n");
         // print column headers
         for (String headerName : headers()) {
-            b.append("\n");
+            addHeader(b, headerName);
         }
+        if (hasUpdateModes)
+            addHeader(b, "Dynamic Update Mode");
         b.append("\n");
         for (ConfigKey key : configs) {
             if (key.internalConfig) {
@@ -1111,10 +1135,15 @@ public String toHtmlTable() {
             b.append("\n");
             // print column values
             for (String headerName : headers()) {
-                b.append("");
             }
+            if (hasUpdateModes) {
+                String updateMode = dynamicUpdateModes.get(key.name);
+                if (updateMode == null)
+                    updateMode = "read-only";
+                addColumnValue(b, updateMode);
+            }
             b.append("\n");
         }
         b.append("
    ");
-            b.append(headerName);
-            b.append("
    ");
-                b.append(getConfigValue(key, headerName));
+                addColumnValue(b, getConfigValue(key, headerName));
                 b.append("
    ");
diff --git a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
index 339c51aa4b12b..affa5dd44367d 100644
--- a/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/config/ConfigDefTest.java
@@ -39,6 +39,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class ConfigDefTest {
@@ -367,6 +368,32 @@ public void testInternalConfigDoesntShowUpInDocs() throws Exception {
         assertFalse(configDef.toRst().contains("my.config"));
     }
 
+    @Test
+    public void testDynamicUpdateModeInDocs() throws Exception {
+        final ConfigDef configDef = new ConfigDef()
+                .define("my.broker.config", Type.LONG, Importance.HIGH, "docs")
+                .define("my.cluster.config", Type.LONG, Importance.HIGH, "docs")
+                .define("my.readonly.config", Type.LONG, Importance.HIGH, "docs");
+        final Map updateModes = new HashMap<>();
+        updateModes.put("my.broker.config", "per-broker");
+        updateModes.put("my.cluster.config", "cluster-wide");
+        final String html = configDef.toHtmlTable(updateModes);
+        Set configsInHtml = new HashSet();
+        for (String line : html.split("\n")) {
+            if (line.contains("my.broker.config")) {
+                assertTrue(line.contains("per-broker"));
+                configsInHtml.add("my.broker.config");
+            } else if (line.contains("my.cluster.config")) {
+                assertTrue(line.contains("cluster-wide"));
+                configsInHtml.add("my.cluster.config");
+            } else if (line.contains("my.readonly.config")) {
+                assertTrue(line.contains("read-only"));
+                configsInHtml.add("my.readonly.config");
+            }
+        }
+        assertEquals(configDef.names(), configsInHtml);
+    }
+
     @Test
     public void testNames() {
         final ConfigDef configDef = new ConfigDef()
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index a95de0aa93161..ce4b9e75b7c53 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -115,6 +115,13 @@ object DynamicBrokerConfig {
         config.displayName, config.dependents, config.recommender)
     }
   }
+
+  private[server] def dynamicConfigUpdateModes: util.Map[String, String] = {
+    AllDynamicConfigs.map { name =>
+      val mode = if (PerBrokerConfigs.contains(name)) "per-broker" else "cluster-wide"
+      (name -> mode)
+    }.toMap.asJava
+  }
 }
 
 class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging {
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 0b9bdaa8bea10..529d0e639d9f0 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -238,7 +238,7 @@ object KafkaConfig {
   private val LogConfigPrefix = "log."
 
   def main(args: Array[String]) {
-    System.out.println(configDef.toHtmlTable)
+    System.out.println(configDef.toHtmlTable(DynamicBrokerConfig.dynamicConfigUpdateModes))
   }
 
   /** ********* Zookeeper Configuration ***********/
diff --git a/docs/configuration.html b/docs/configuration.html
index df5eebb3ee954..df58ba76b773f 100644
--- a/docs/configuration.html
+++ b/docs/configuration.html
@@ -33,6 +33,162 @@ 
    3.1 Broker Configs
    
 
   
    More details about broker configuration can be found in the scala class kafka.server.KafkaConfig.
    
 
+  
    3.1.1 Updating Broker Configs
    
+  From Kafka version 1.1 onwards, some of the broker configs can be updated without restarting the broker. See the
+  Dynamic Update Mode column in Broker Configs for the update mode of each broker config.
+  
      
+    
    • read-only: Requires a broker restart for update
      • 
+    
    • per-broker: May be updated dynamically for each broker
      • 
+    
    • cluster-wide: May be updated dynamically as a cluster-wide default. May also be updated as a per-broker value for testing.
      • 
+  
    
+
+  To alter the current broker configs for broker id 0 (for example, the number of log cleaner threads):
+  
    +  > bin/kafka-configs.sh --bootstrap-server localhost:9092 --entity-type brokers --entity-name 0 --alter --add-config log.cleaner.threads=2
+  
    
+
+  To describe the current dynamic broker configs for broker id 0:
+  
    +  > bin/kafka-configs.sh --bootstrap-server localhost:9092 --entity-type brokers --entity-name 0 --describe
+  
    
+
+  To delete a config override and revert to the statically configured or default value for broker id 0 (for example,
+  the number of log cleaner threads):
+  
    +  > bin/kafka-configs.sh --bootstrap-server localhost:9092 --entity-type brokers --entity-name 0 --alter --delete-config log.cleaner.threads
+  
    
+
+  Some configs may be configured as a cluster-wide default to maintain consistent values across the whole cluster.  All brokers
+  in the cluster will process the cluster default update. For example, to update log cleaner threads on all brokers:
+  
    +  > bin/kafka-configs.sh --bootstrap-server localhost:9092 --entity-type brokers --entity-default --alter --add-config log.cleaner.threads=2
+  
    
+
+  To describe the currently configured dynamic cluster-wide default configs:
+  
    +  > bin/kafka-configs.sh --bootstrap-server localhost:9092 --entity-type brokers --entity-default --describe
+  
    
+
+  All configs that are configurable at cluster level may also be configured at per-broker level (e.g. for testing).
+  If a config value is defined at different levels, the following order of precedence is used:
+  
      
+  
    • Dynamic per-broker config stored in ZooKeeper
      • 
+  
    • Dynamic cluster-wide default config stored in ZooKeeper
      • 
+  
    • Static broker config from server.properties
      • 
+  
    • Kafka default, see broker configs
      • 
+  
    
+
+  
    Updating Password Configs Dynamically
    
+  
    Password config values that are dynamically updated are encrypted before storing in ZooKeeper. The broker config
+  password.encoder.secret must be configured in server.properties to enable dynamic update
+  of password configs. The secret may be different on different brokers.
    
+  
    The secret used for password encoding may be rotated with a rolling restart of brokers. The old secret used for encoding
+  passwords currently in ZooKeeper must be provided in the static broker config password.encoder.old.secret and
+  the new secret must be provided in password.encoder.secret. All dynamic password configs stored in ZooKeeper
+  will be re-encoded with the new secret when the broker starts up.
    
+  
    In Kafka 1.1.x, all dynamically updated password configs must be provided in every alter request when updating configs
+  using kafka-configs.sh even if the password config is not being altered. This constraint will be removed in
+  a future release.
    
+
+  
    Updating SSL Keystore of an Existing Listener
    
+  Brokers may be configured with SSL keystores with short validity periods to reduce the risk of compromised certificates.
+  Keystores may be updated dynamically without restarting the broker. The config name must be prefixed with the listener prefix
+  listener.name.{listenerName}. so that only the keystore config of a specific listener is updated.
+  The following configs may be updated in a single alter request at per-broker level:
+  
      
+    
    • ssl.keystore.type
      • 
+    
    • ssl.keystore.location
      • 
+    
    • ssl.keystore.password
      • 
+    
    • ssl.key.password
      • 
+  
    
+  If the listener is the inter-broker listener, the update is allowed only if the new keystore is trusted by the truststore
+  configured for that listener. For other listeners, no trust validation is performed on the keystore by the broker. Certificates
+  must be signed by the same certificate authority that signed the old certificate to avoid any client authentication failures.
+
+  
    Updating Default Topic Configuration
    
+  Default topic configuration options used by brokers may be updated without broker restart. The configs are applied to topics
+  without a topic config override for the equivalent per-topic config. One or more of these configs may be overridden at
+  cluster-default level used by all brokers.
+  
      
+    
    • log.segment.bytes
      • 
+    
    • log.roll.ms
      • 
+    
    • log.roll.hours
      • 
+    
    • log.roll.jitter.ms
      • 
+    
    • log.roll.jitter.hours
      • 
+    
    • log.index.size.max.bytes
      • 
+    
    • log.flush.interval.messages
      • 
+    
    • log.flush.interval.ms
      • 
+    
    • log.retention.bytes
      • 
+    
    • log.retention.ms
      • 
+    
    • log.retention.minutes
      • 
+    
    • log.retention.hours
      • 
+    
    • log.index.interval.bytes
      • 
+    
    • log.cleaner.delete.retention.ms
      • 
+    
    • log.cleaner.min.compaction.lag.ms
      • 
+    
    • log.cleaner.min.cleanable.ratio
      • 
+    
    • log.cleanup.policy
      • 
+    
    • log.segment.delete.delay.ms
      • 
+    
    • unclean.leader.election.enable
      • 
+    
    • min.insync.replicas
      • 
+    
    • max.message.bytes
      • 
+    
    • compression.type
      • 
+    
    • log.preallocate
      • 
+    
    • log.message.timestamp.type
      • 
+    
    • log.message.timestamp.difference.max.ms
      • 
+  
    
+
+  In Kafka version 1.1.x, changes to unclean.leader.election.enable take effect only when a new controller is elected.
+  Controller re-election may be forced by running:
+
+  
    +  > bin/zookeeper-shell.sh localhost
+  rmr /controller
+  
    
+
+  
    Updating Log Cleaner Configs
    
+  Log cleaner configs may be updated dynamically at cluster-default level used by all brokers. The changes take effect
+  on the next iteration of log cleaning. One or more of these configs may be updated:
+  
      
+    
    • log.cleaner.threads
      • 
+    
    • log.cleaner.io.max.bytes.per.second
      • 
+    
    • log.cleaner.dedupe.buffer.size
      • 
+    
    • log.cleaner.io.buffer.size
      • 
+    
    • log.cleaner.io.buffer.load.factor
      • 
+    
    • log.cleaner.backoff.ms
      • 
+  
    
+
+  
    Updating Thread Configs
    
+  The size of various thread pools used by the broker may be updated dynamically at cluster-default level used by all brokers.
+  Updates are restricted to the range currentSize / 2 to currentSize * 2 to ensure that config updates are
+  handled gracefully.
+  
      
+    
    • num.network.threads
      • 
+    
    • num.io.threads
      • 
+    
    • num.replica.fetchers
      • 
+    
    • num.recovery.threads.per.data.dir
      • 
+    
    • log.cleaner.threads
      • 
+    
    • background.threads
      • 
+  
    
+
+  
    Adding and Removing Listeners
    
+  
    Listeners may be added or removed dynamically. When a new listener is added, security configs of the listener must be provided
+  as listener configs with the listener prefix listener.name.{listenerName}.. If the new listener uses SASL,
+  the JAAS configuration of the listener must be provided using the JAAS configuration property sasl.jaas.config
+  with the listener and mechanism prefix. See JAAS configuration for Kafka brokers for details.
    
+
+  
    In Kafka version 1.1.x, the listener used by the inter-broker listener may not be updated dynamically. To update the inter-broker
+  listener to a new listener, the new listener may be added on all brokers without restarting the broker. A rolling restart is then
+  required to update inter.broker.listener.name.
    
+
+  In addition to all the security configs of new listeners, the following configs may be updated dynamically at per-broker level:
+  
      
+    
    • listeners
      • 
+    
    • advertised.listeners
      • 
+    
    • listener.security.protocol.map
      • 
+  
    
+  Inter-broker listener must be configured using the static broker configuration inter.broker.listener.name
+  or inter.broker.security.protocol.
+
   
    3.2 Topic-Level Configs
    
 
   Configurations pertinent to topics have both a server default as well an optional per-topic override. If no per-topic configuration is given the server default is used. The override can be set at topic creation time by giving one or more --config options. This example creates a topic named my-topic with a custom max message size and flush rate:
diff --git a/docs/security.html b/docs/security.html
index 3e3c818571aba..db6f487377af6 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -231,7 +231,9 @@ 
    7.3 Authentication using SASLKafkaServer is the section name in the JAAS file used by each
             KafkaServer/Broker. This section provides SASL configuration options
             for the broker including any SASL client connections made by the broker
-            for inter-broker communication.
    
+            for inter-broker communication. If multiple listeners are configured to use
+            SASL, the section name may be prefixed with the listener name in lower-case
+            followed by a period, e.g. sasl_ssl.KafkaServer.
    
 
             
    Client section is used to authenticate a SASL connection with
             zookeeper. It also allows the brokers to set SASL ACL on zookeeper
@@ -246,6 +248,35 @@ 
    7.3 Authentication using SASLzookeeper.sasl.client.username to the appropriate name
             (e.g., -Dzookeeper.sasl.client.username=zk).
    
 
+            
    Brokers may also configure JAAS using the broker configuration property sasl.jaas.config.
+            The property name must be prefixed with the listener prefix including the SASL mechanism,
+            i.e. listener.name.{listenerName}.{saslMechanism}.sasl.jaas.config. Only one
+            login module may be specified in the config value. If multiple mechanisms are configured on a
+            listener, configs must be provided for each mechanism using the listener and mechanism prefix.
+            For example,
+                    
    +        listener.name.sasl_ssl.scram-sha-256.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \
+            username="admin" \
+            password="admin-secret";
+        listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+            username="admin" \
+            password="admin-secret" \
+            user_admin="admin-secret" \
+            user_alice="alice-secret";
    
+
+            If JAAS configuration is defined at different levels, the order of precedence used is:
+            
      
+              
    • Broker configuration property listener.name.{listenerName}.{saslMechanism}.sasl.jaas.config
      • 
+              
    • {listenerName}.KafkaServer section of static JAAS configuration
      • 
+              
    • KafkaServer section of static JAAS configuration
      • 
+            
    
+            Note that ZooKeeper JAAS config may only be configured using static JAAS configuration.
+
+            
    See GSSAPI (Kerberos),
+            PLAIN or
+            SCRAM for example broker configurations.
    
+
+
         
  • JAAS configuration for Kafka clients
    
 

From 3af13967db089b9a8320b539f5d5d218488ce467 Mon Sep 17 00:00:00 2001
From: Robert Yokota 
Date: Wed, 14 Feb 2018 16:24:05 -0800
Subject: [PATCH 0055/1847] KAFKA-6503: Parallelize plugin scanning

This is a small change to parallelize plugin scanning.  This may help in some environments where otherwise plugin scanning is slow.

Author: Robert Yokota 

Reviewers: Konstantine Karantasis , Randall Hauch , Ewen Cheslack-Postava 

Closes #4561 from rayokota/K6503-improve-plugin-scanning
---
 .../isolation/DelegatingClassLoader.java      | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
index 345d7ef011dac..b21cdcbfab448 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
@@ -20,7 +20,10 @@
 import org.apache.kafka.connect.storage.Converter;
 import org.apache.kafka.connect.storage.HeaderConverter;
 import org.apache.kafka.connect.transforms.Transformation;
+import org.reflections.Configuration;
 import org.reflections.Reflections;
+import org.reflections.ReflectionsException;
+import org.reflections.scanners.SubTypesScanner;
 import org.reflections.util.ClasspathHelper;
 import org.reflections.util.ConfigurationBuilder;
 import org.slf4j.Logger;
@@ -269,7 +272,10 @@ private PluginScanResult scanPluginPath(
         ConfigurationBuilder builder = new ConfigurationBuilder();
         builder.setClassLoaders(new ClassLoader[]{loader});
         builder.addUrls(urls);
-        Reflections reflections = new Reflections(builder);
+        builder.setScanners(new SubTypesScanner());
+        builder.setExpandSuperTypes(false);
+        builder.useParallelExecutor();
+        Reflections reflections = new InternalReflections(builder);
 
         return new PluginScanResult(
                 getPluginDesc(reflections, Connector.class, loader),
@@ -353,4 +359,25 @@ private  void addAliases(Collection> plugins) {
             }
         }
     }
+
+    private static class InternalReflections extends Reflections {
+
+        public InternalReflections(Configuration configuration) {
+            super(configuration);
+        }
+
+        // When Reflections is used for parallel scans, it has a bug where it propagates ReflectionsException
+        // as RuntimeException.  Override the scan behavior to emulate the singled-threaded logic.
+        @Override
+        protected void scan(URL url) {
+            try {
+                super.scan(url);
+            } catch (ReflectionsException e) {
+                Logger log = Reflections.log;
+                if (log != null && log.isWarnEnabled()) {
+                    log.warn("could not create Vfs.Dir from url. ignoring the exception and continuing", e);
+                }
+            }
+        }
+    }
 }

From 50cd85538580c9c9833ad957d64d39953c124374 Mon Sep 17 00:00:00 2001
From: Sebastian Bauersfeld 
Date: Thu, 15 Feb 2018 01:02:47 +0000
Subject: [PATCH 0056/1847] HOTFIX: Fix lgtm.com alerts (dead code and
 out-of-bounds error) (#4388)

This fixes two alerts flagged on lgtm.com for Apache Kafka.

This dead code alert where InvalidTypeIdException indirectly extends JsonMappingException. The flagged condition with the type test appears after the type test for the latter and thus makes its body dead. I opted to change the order of the tests. Please let me know if this is the intended behavior.

The second commit addresses this out-of-bounds alert.

More alerts can be found here. Note that my colleague Aditya Sharad addressed some of those in the now outdated #2939.

Reviewers: Matthias J. Sax , Rajini Sivaram 
---
 .../security/kerberos/KerberosRule.java       |  6 +--
 .../security/kerberos/KerberosRuleTest.java   | 49 +++++++++++++++++++
 2 files changed, 52 insertions(+), 3 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosRuleTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
index 37820df5aa2d6..92a70b1f19456 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
@@ -107,8 +107,8 @@ public String toString() {
     }
 
     /**
-     * Replace the numbered parameters of the form $n where n is from 1 to
-     * the length of params. Normal text is copied directly and $n is replaced
+     * Replace the numbered parameters of the form $n where n is from 0 to
+     * the length of params - 1. Normal text is copied directly and $n is replaced
      * by the corresponding parameter.
      * @param format the string to replace parameters again
      * @param params the list of parameters
@@ -126,7 +126,7 @@ static String replaceParameters(String format,
             if (paramNum != null) {
                 try {
                     int num = Integer.parseInt(paramNum);
-                    if (num < 0 || num > params.length) {
+                    if (num < 0 || num >= params.length) {
                         throw new BadFormatString("index " + num + " from " + format +
                                 " is outside of the valid range 0 to " +
                                 (params.length - 1));
diff --git a/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosRuleTest.java b/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosRuleTest.java
new file mode 100644
index 0000000000000..f79c47af05fdb
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/security/kerberos/KerberosRuleTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.security.kerberos;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+
+public class KerberosRuleTest {
+
+    @Test
+    public void testReplaceParameters() throws BadFormatString {
+        // positive test cases
+        assertEquals(KerberosRule.replaceParameters("", new String[0]), "");
+        assertEquals(KerberosRule.replaceParameters("hello", new String[0]), "hello");
+        assertEquals(KerberosRule.replaceParameters("", new String[]{"too", "many", "parameters", "are", "ok"}), "");
+        assertEquals(KerberosRule.replaceParameters("hello", new String[]{"too", "many", "parameters", "are", "ok"}), "hello");
+        assertEquals(KerberosRule.replaceParameters("hello $0", new String[]{"too", "many", "parameters", "are", "ok"}), "hello too");
+        assertEquals(KerberosRule.replaceParameters("hello $0", new String[]{"no recursion $1"}), "hello no recursion $1");
+
+        // negative test cases
+        try {
+            KerberosRule.replaceParameters("$0", new String[]{});
+            fail("An out-of-bounds parameter number should trigger an exception!");
+        } catch (BadFormatString bfs) {
+        }
+        try {
+            KerberosRule.replaceParameters("hello $a", new String[]{"does not matter"});
+            fail("A malformed parameter name should trigger an exception!");
+        } catch (BadFormatString bfs) {
+        }
+    }
+
+}

From 88307657dff0202804447d06eef6ecdc808590d8 Mon Sep 17 00:00:00 2001
From: Lucas Wang 
Date: Wed, 14 Feb 2018 17:30:09 -0800
Subject: [PATCH 0057/1847] =?UTF-8?q?KAFKA-6481;=20Improving=20performance?=
 =?UTF-8?q?=20of=20the=20function=20ControllerChannelManager.addUpd?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ateMetadataRequestForBrokers

*More detailed description of your change,
if necessary. The PR title and PR message become
the squashed commit message, so use a separate
comment to ping reviewers.*

*Summary of testing strategy (including rationale)
for the feature or bug fix. Unit and/or integration
tests are expected for any behaviour change and
system tests should be considered for larger changes.*

Author: Lucas Wang 

Reviewers: Jun Rao 

Closes #4472 from gitlw/improving_addUpdateMetadataRequestForBrokers
---
 .../controller/ControllerChannelManager.scala  | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index d5456bea9a8cf..4d7ccf1247f4a 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -375,20 +375,14 @@ class ControllerBrokerRequestBatch(controller: KafkaController, stateChangeLogge
       }
     }
 
-    val filteredPartitions = {
-      val givenPartitions = if (partitions.isEmpty)
-        controllerContext.partitionLeadershipInfo.keySet
-      else
-        partitions
-      if (controller.topicDeletionManager.partitionsToBeDeleted.isEmpty)
-        givenPartitions
-      else
-        givenPartitions -- controller.topicDeletionManager.partitionsToBeDeleted
-    }
+    val givenPartitions = if (partitions.isEmpty)
+      controllerContext.partitionLeadershipInfo.keySet
+    else
+      partitions
 
     updateMetadataRequestBrokerSet ++= brokerIds.filter(_ >= 0)
-    filteredPartitions.foreach(partition => updateMetadataRequestPartitionInfo(partition, beingDeleted = false))
-    controller.topicDeletionManager.partitionsToBeDeleted.foreach(partition => updateMetadataRequestPartitionInfo(partition, beingDeleted = true))
+    givenPartitions.foreach(partition => updateMetadataRequestPartitionInfo(partition,
+      beingDeleted = controller.topicDeletionManager.partitionsToBeDeleted.contains(partition)))
   }
 
   def sendRequestsToBrokers(controllerEpoch: Int) {

From 015e224b3d3c8e7bc412686ff22d5e99324b1019 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 15 Feb 2018 09:26:34 +0000
Subject: [PATCH 0058/1847] MINOR: Support dynamic JAAS config for broker's
 LoginManager cache (#4568)

Fix LoginManager caching when sasl.jaas.config is defined for broker and add unit tests.

Reviewers: Jason Gustafson 
---
 .../kafka/common/security/JaasContext.java    |  28 ++--
 .../security/authenticator/LoginManager.java  |  34 +++--
 .../network/SaslChannelBuilderTest.java       |   2 +-
 .../ClientAuthenticationFailureTest.java      |   1 +
 .../authenticator/LoginManagerTest.java       | 129 ++++++++++++++++++
 .../authenticator/SaslAuthenticatorTest.java  |   1 +
 .../SaslServerAuthenticatorTest.java          |   2 +-
 .../security/plain/PlainSaslServerTest.java   |   2 +-
 8 files changed, 177 insertions(+), 22 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java b/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java
index 6afed5569251f..d72f00dc590d5 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java
@@ -64,10 +64,10 @@ public static JaasContext loadServerContext(ListenerName listenerName, String me
             throw new IllegalArgumentException("mechanism should not be null for SERVER");
         String globalContextName = GLOBAL_CONTEXT_NAME_SERVER;
         String listenerContextName = listenerName.value().toLowerCase(Locale.ROOT) + "." + GLOBAL_CONTEXT_NAME_SERVER;
-        Password jaasConfigArgs = (Password) configs.get(mechanism.toLowerCase(Locale.ROOT) + "." + SaslConfigs.SASL_JAAS_CONFIG);
-        if (jaasConfigArgs == null && configs.get(SaslConfigs.SASL_JAAS_CONFIG) != null)
+        Password dynamicJaasConfig = (Password) configs.get(mechanism.toLowerCase(Locale.ROOT) + "." + SaslConfigs.SASL_JAAS_CONFIG);
+        if (dynamicJaasConfig == null && configs.get(SaslConfigs.SASL_JAAS_CONFIG) != null)
             LOG.warn("Server config {} should be prefixed with SASL mechanism name, ignoring config", SaslConfigs.SASL_JAAS_CONFIG);
-        return load(Type.SERVER, listenerContextName, globalContextName, jaasConfigArgs);
+        return load(Type.SERVER, listenerContextName, globalContextName, dynamicJaasConfig);
     }
 
     /**
@@ -80,20 +80,20 @@ public static JaasContext loadServerContext(ListenerName listenerName, String me
      */
     public static JaasContext loadClientContext(Map configs) {
         String globalContextName = GLOBAL_CONTEXT_NAME_CLIENT;
-        Password jaasConfigArgs = (Password) configs.get(SaslConfigs.SASL_JAAS_CONFIG);
-        return load(JaasContext.Type.CLIENT, null, globalContextName, jaasConfigArgs);
+        Password dynamicJaasConfig = (Password) configs.get(SaslConfigs.SASL_JAAS_CONFIG);
+        return load(JaasContext.Type.CLIENT, null, globalContextName, dynamicJaasConfig);
     }
 
     static JaasContext load(JaasContext.Type contextType, String listenerContextName,
-                            String globalContextName, Password jaasConfigArgs) {
-        if (jaasConfigArgs != null) {
-            JaasConfig jaasConfig = new JaasConfig(globalContextName, jaasConfigArgs.value());
+                            String globalContextName, Password dynamicJaasConfig) {
+        if (dynamicJaasConfig != null) {
+            JaasConfig jaasConfig = new JaasConfig(globalContextName, dynamicJaasConfig.value());
             AppConfigurationEntry[] contextModules = jaasConfig.getAppConfigurationEntry(globalContextName);
             if (contextModules == null || contextModules.length == 0)
                 throw new IllegalArgumentException("JAAS config property does not contain any login modules");
             else if (contextModules.length != 1)
                 throw new IllegalArgumentException("JAAS config property contains " + contextModules.length + " login modules, should be 1 module");
-            return new JaasContext(globalContextName, contextType, jaasConfig);
+            return new JaasContext(globalContextName, contextType, jaasConfig, dynamicJaasConfig);
         } else
             return defaultContext(contextType, listenerContextName, globalContextName);
     }
@@ -133,7 +133,7 @@ private static JaasContext defaultContext(JaasContext.Type contextType, String l
             throw new IllegalArgumentException(errorMessage);
         }
 
-        return new JaasContext(contextName, contextType, jaasConfig);
+        return new JaasContext(contextName, contextType, jaasConfig, null);
     }
 
     /**
@@ -146,8 +146,9 @@ public enum Type { CLIENT, SERVER; }
     private final Type type;
     private final Configuration configuration;
     private final List configurationEntries;
+    private final Password dynamicJaasConfig;
 
-    public JaasContext(String name, Type type, Configuration configuration) {
+    public JaasContext(String name, Type type, Configuration configuration, Password dynamicJaasConfig) {
         this.name = name;
         this.type = type;
         this.configuration = configuration;
@@ -155,6 +156,7 @@ public JaasContext(String name, Type type, Configuration configuration) {
         if (entries == null)
             throw new IllegalArgumentException("Could not find a '" + name + "' entry in this JAAS configuration.");
         this.configurationEntries = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(entries)));
+        this.dynamicJaasConfig = dynamicJaasConfig;
     }
 
     public String name() {
@@ -173,6 +175,10 @@ public List configurationEntries() {
         return configurationEntries;
     }
 
+    public Password dynamicJaasConfig() {
+        return dynamicJaasConfig;
+    }
+
     /**
      * Returns the configuration option for key from this context.
      * If login module name is specified, return option value only from that module.
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java
index dc264c8a5fdbc..81dc063c7433f 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java
@@ -46,8 +46,8 @@ public class LoginManager {
     private int refCount;
 
     private LoginManager(JaasContext jaasContext, boolean hasKerberos, Map configs,
-                         Password jaasConfigValue) throws IOException, LoginException {
-        this.cacheKey = jaasConfigValue != null ? jaasConfigValue : jaasContext.name();
+                         Object cacheKey) throws IOException, LoginException {
+        this.cacheKey = cacheKey;
         login = hasKerberos ? new KerberosLogin() : new DefaultLogin();
         login.configure(configs, jaasContext);
         login.login();
@@ -57,20 +57,33 @@ private LoginManager(JaasContext jaasContext, boolean hasKerberos, Map configs) throws IOException, LoginException {
         synchronized (LoginManager.class) {
-            // SASL_JAAS_CONFIG is only supported by clients
             LoginManager loginManager;
-            Password jaasConfigValue = (Password) configs.get(SaslConfigs.SASL_JAAS_CONFIG);
-            if (jaasContext.type() == JaasContext.Type.CLIENT && jaasConfigValue != null) {
+            Password jaasConfigValue = jaasContext.dynamicJaasConfig();
+            if (jaasConfigValue != null) {
                 loginManager = DYNAMIC_INSTANCES.get(jaasConfigValue);
                 if (loginManager == null) {
                     loginManager = new LoginManager(jaasContext, saslMechanism.equals(SaslConfigs.GSSAPI_MECHANISM), configs, jaasConfigValue);
@@ -79,7 +92,7 @@ public static LoginManager acquireLoginManager(JaasContext jaasContext, String s
             } else {
                 loginManager = STATIC_INSTANCES.get(jaasContext.name());
                 if (loginManager == null) {
-                    loginManager = new LoginManager(jaasContext, hasKerberos, configs, jaasConfigValue);
+                    loginManager = new LoginManager(jaasContext, hasKerberos, configs, jaasContext.name());
                     STATIC_INSTANCES.put(jaasContext.name(), loginManager);
                 }
             }
@@ -95,6 +108,11 @@ public String serviceName() {
         return login.serviceName();
     }
 
+    // Only for testing
+    Object cacheKey() {
+        return cacheKey;
+    }
+
     private LoginManager acquire() {
         ++refCount;
         LOGGER.trace("{} acquired", this);
diff --git a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java
index 6072bf54fa7b2..26cc544cd6d8b 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java
@@ -71,7 +71,7 @@ public void testLoginManagerReleasedIfConfigureThrowsException() {
     private SaslChannelBuilder createChannelBuilder(SecurityProtocol securityProtocol) {
         TestJaasConfig jaasConfig = new TestJaasConfig();
         jaasConfig.addEntry("jaasContext", PlainLoginModule.class.getName(), new HashMap());
-        JaasContext jaasContext = new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig);
+        JaasContext jaasContext = new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig, null);
         Map jaasContexts = Collections.singletonMap("PLAIN", jaasContext);
         return new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, new ListenerName("PLAIN"),
                 false, "PLAIN", true, null, null);
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java
index 7c028c4db6171..c64597b7e8b35 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java
@@ -56,6 +56,7 @@ public class ClientAuthenticationFailureTest {
 
     @Before
     public void setup() throws Exception {
+        LoginManager.closeAll();
         SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT;
 
         saslServerConfigs = new HashMap<>();
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java
new file mode 100644
index 0000000000000..8be72fb5b8c93
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.security.authenticator;
+
+import org.apache.kafka.common.config.types.Password;
+import org.apache.kafka.common.network.ListenerName;
+import org.apache.kafka.common.security.JaasContext;
+import org.apache.kafka.common.security.plain.PlainLoginModule;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertEquals;
+
+public class LoginManagerTest {
+
+    private Password dynamicPlainContext;
+    private Password dynamicDigestContext;
+
+    @Before
+    public void setUp() {
+        dynamicPlainContext = new Password(PlainLoginModule.class.getName() +
+                " required user=\"plainuser\" password=\"plain-secret\";");
+        dynamicDigestContext = new Password(TestDigestLoginModule.class.getName() +
+                " required user=\"digestuser\" password=\"digest-secret\";");
+        TestJaasConfig.createConfiguration("SCRAM-SHA-256",
+                Collections.singletonList("SCRAM-SHA-256"));
+    }
+
+    @After
+    public void tearDown() {
+        LoginManager.closeAll();
+    }
+
+    @Test
+    public void testClientLoginManager() throws Exception {
+        Map configs = Collections.singletonMap("sasl.jaas.config", dynamicPlainContext);
+        JaasContext dynamicContext = JaasContext.loadClientContext(configs);
+        JaasContext staticContext = JaasContext.loadClientContext(Collections.emptyMap());
+
+        LoginManager dynamicLogin = LoginManager.acquireLoginManager(dynamicContext, "PLAIN",
+                false, configs);
+        assertEquals(dynamicPlainContext, dynamicLogin.cacheKey());
+        LoginManager staticLogin = LoginManager.acquireLoginManager(staticContext, "SCRAM-SHA-256",
+                false, configs);
+        assertNotSame(dynamicLogin, staticLogin);
+        assertEquals("KafkaClient", staticLogin.cacheKey());
+
+        assertSame(dynamicLogin, LoginManager.acquireLoginManager(dynamicContext, "PLAIN",
+                false, configs));
+        assertSame(staticLogin, LoginManager.acquireLoginManager(staticContext, "SCRAM-SHA-256",
+                false, configs));
+
+        verifyLoginManagerRelease(dynamicLogin, 2, dynamicContext, configs);
+        verifyLoginManagerRelease(staticLogin, 2, staticContext, configs);
+    }
+
+    @Test
+    public void testServerLoginManager() throws Exception {
+        Map configs = new HashMap<>();
+        configs.put("plain.sasl.jaas.config", dynamicPlainContext);
+        configs.put("digest-md5.sasl.jaas.config", dynamicDigestContext);
+        ListenerName listenerName = new ListenerName("listener1");
+        JaasContext plainJaasContext = JaasContext.loadServerContext(listenerName, "PLAIN", configs);
+        JaasContext digestJaasContext = JaasContext.loadServerContext(listenerName, "DIGEST-MD5", configs);
+        JaasContext scramJaasContext = JaasContext.loadServerContext(listenerName, "SCRAM-SHA-256", configs);
+
+        LoginManager dynamicPlainLogin = LoginManager.acquireLoginManager(plainJaasContext, "PLAIN",
+                false, configs);
+        assertEquals(dynamicPlainContext, dynamicPlainLogin.cacheKey());
+        LoginManager dynamicDigestLogin = LoginManager.acquireLoginManager(digestJaasContext, "DIGEST-MD5",
+                false, configs);
+        assertNotSame(dynamicPlainLogin, dynamicDigestLogin);
+        assertEquals(dynamicDigestContext, dynamicDigestLogin.cacheKey());
+        LoginManager staticScramLogin = LoginManager.acquireLoginManager(scramJaasContext, "SCRAM-SHA-256",
+                false, configs);
+        assertNotSame(dynamicPlainLogin, staticScramLogin);
+        assertEquals("KafkaServer", staticScramLogin.cacheKey());
+
+        assertSame(dynamicPlainLogin, LoginManager.acquireLoginManager(plainJaasContext, "PLAIN",
+                false, configs));
+        assertSame(dynamicDigestLogin, LoginManager.acquireLoginManager(digestJaasContext, "DIGEST-MD5",
+                false, configs));
+        assertSame(staticScramLogin, LoginManager.acquireLoginManager(scramJaasContext, "SCRAM-SHA-256",
+                false, configs));
+
+        verifyLoginManagerRelease(dynamicPlainLogin, 2, plainJaasContext, configs);
+        verifyLoginManagerRelease(dynamicDigestLogin, 2, digestJaasContext, configs);
+        verifyLoginManagerRelease(staticScramLogin, 2, scramJaasContext, configs);
+    }
+
+    private void verifyLoginManagerRelease(LoginManager loginManager, int acquireCount, JaasContext jaasContext,
+                                           Map configs) throws Exception {
+
+        // Release all except one reference and verify that the loginManager is still cached
+        for (int i = 0; i < acquireCount - 1; i++)
+            loginManager.release();
+        assertSame(loginManager, LoginManager.acquireLoginManager(jaasContext, "PLAIN",
+                false, configs));
+
+        // Release all references and verify that new LoginManager is created on next acquire
+        for (int i = 0; i < 2; i++) // release all references
+            loginManager.release();
+        LoginManager newLoginManager = LoginManager.acquireLoginManager(jaasContext, "PLAIN",
+                false, configs);
+        assertNotSame(loginManager, newLoginManager);
+        newLoginManager.release();
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index ef2a07578daf1..b8edc612f6975 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -104,6 +104,7 @@ public class SaslAuthenticatorTest {
 
     @Before
     public void setup() throws Exception {
+        LoginManager.closeAll();
         serverCertStores = new CertStores(true, "localhost");
         clientCertStores = new CertStores(false, "localhost");
         saslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java
index 72c296943e719..17d31bda72ccb 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java
@@ -110,7 +110,7 @@ private SaslServerAuthenticator setupAuthenticator(Map configs, Trans
         TestJaasConfig jaasConfig = new TestJaasConfig();
         jaasConfig.addEntry("jaasContext", PlainLoginModule.class.getName(), new HashMap());
         Map jaasContexts = Collections.singletonMap(mechanism,
-                new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig));
+                new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig, null));
         Map subjects = Collections.singletonMap(mechanism, new Subject());
         return new SaslServerAuthenticator(configs, "node", jaasContexts, subjects, null, new CredentialCache(),
                 new ListenerName("ssl"), SecurityProtocol.SASL_SSL, transportLayer, new DelegationTokenCache(ScramMechanism.mechanismNames()));
diff --git a/clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java b/clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java
index 4196db6ac0f53..86baf3e07af23 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java
@@ -45,7 +45,7 @@ public void setUp() throws Exception {
         options.put("user_" + USER_A, PASSWORD_A);
         options.put("user_" + USER_B, PASSWORD_B);
         jaasConfig.addEntry("jaasContext", PlainLoginModule.class.getName(), options);
-        JaasContext jaasContext = new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig);
+        JaasContext jaasContext = new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig, null);
         saslServer = new PlainSaslServer(jaasContext);
     }
 

From d9d0d79287eeec0a1c3dcc2203288421284b5ca1 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Thu, 15 Feb 2018 22:11:38 +0530
Subject: [PATCH 0059/1847] MINOR: Update test classes to use KafkaZkClient
 methods (#4367)

Remove ZkUtils reference form ZooKeeperTestHarness plus some minor cleanups.
---
 core/src/main/scala/kafka/utils/ZkUtils.scala |  2 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   | 22 ++++-
 .../kafka/api/AuthorizerIntegrationTest.scala | 12 +--
 .../kafka/api/ConsumerBounceTest.scala        |  2 +-
 .../api/ProducerFailureHandlingTest.scala     | 16 ++--
 .../api/SaslPlainPlaintextConsumerTest.scala  |  5 +-
 ...aslPlainSslEndToEndAuthorizationTest.scala |  5 +-
 ...aslScramSslEndToEndAuthorizationTest.scala |  4 +-
 .../scala/unit/kafka/admin/AdminTest.scala    | 34 ++++---
 .../kafka/admin/DeleteConsumerGroupTest.scala | 18 +++-
 .../unit/kafka/admin/TopicCommandTest.scala   |  8 +-
 .../ZookeeperConsumerConnectorTest.scala      |  7 +-
 .../ControllerIntegrationTest.scala           | 92 +++++++++----------
 .../unit/kafka/integration/FetcherTest.scala  | 12 ++-
 .../security/auth/ZkAuthorizationTest.scala   | 11 ++-
 .../kafka/server/LeaderElectionTest.scala     |  8 +-
 .../unit/kafka/server/LogDirFailureTest.scala |  4 +-
 .../server/ServerGenerateBrokerIdTest.scala   |  8 --
 .../server/ServerGenerateClusterIdTest.scala  |  6 +-
 .../unit/kafka/server/ServerStartupTest.scala |  6 +-
 .../kafka/utils/ReplicationUtilsTest.scala    | 15 +--
 .../scala/unit/kafka/utils/TestUtils.scala    | 15 +--
 .../scala/unit/kafka/utils/ZkUtilsTest.scala  | 24 ++++-
 .../scala/unit/kafka/zk/ZKEphemeralTest.scala | 13 +--
 .../unit/kafka/zk/ZooKeeperTestHarness.scala  |  6 +-
 25 files changed, 216 insertions(+), 139 deletions(-)

diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala
index e04bce0c38fc7..d5fde4db559bc 100644
--- a/core/src/main/scala/kafka/utils/ZkUtils.scala
+++ b/core/src/main/scala/kafka/utils/ZkUtils.scala
@@ -592,7 +592,7 @@ class ZkUtils(val zkClient: ZkClient,
   }
 
   /**
-   * Update the value of a persistent node with the given path and data.
+   * Update the value of a ephemeral node with the given path and data.
    * create parent directory if necessary. Never throw NodeExistException.
    */
   def updateEphemeralPath(path: String, data: String, acls: java.util.List[ACL] = UseDefaultAcls): Unit = {
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index b545455ec52d0..afc8202b88a38 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -761,12 +761,30 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   /**
    * Gets the leader for a given partition
-   * @param partition
+   * @param partition The partition for which we want to get leader.
    * @return optional integer if the leader exists and None otherwise.
    */
   def getLeaderForPartition(partition: TopicPartition): Option[Int] =
     getTopicPartitionState(partition).map(_.leaderAndIsr.leader)
 
+  /**
+   * Gets the in-sync replicas (ISR) for a specific topicPartition
+   * @param partition The partition for which we want to get ISR.
+   * @return optional ISR if exists and None otherwise
+   */
+  def getInSyncReplicasForPartition(partition: TopicPartition): Option[Seq[Int]] =
+    getTopicPartitionState(partition).map(_.leaderAndIsr.isr)
+
+
+  /**
+   * Gets the leader epoch for a specific topicPartition
+   * @param partition The partition for which we want to get the leader epoch
+   * @return optional integer if the leader exists and None otherwise
+   */
+  def getEpochForPartition(partition: TopicPartition): Option[Int] = {
+    getTopicPartitionState(partition).map(_.leaderAndIsr.leaderEpoch)
+  }
+
   /**
    * Gets the isr change notifications as strings. These strings are the znode names and not the absolute znode path.
    * @return sequence of znode names and not the absolute znode path.
@@ -1356,7 +1374,7 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
     }
   }
 
-  private[zk] def pathExists(path: String): Boolean = {
+  def pathExists(path: String): Boolean = {
     val existsRequest = ExistsRequest(path)
     val existsResponse = retryRequestUntilConnected(existsRequest)
     existsResponse.resultCode match {
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 248d219cf31d5..ab7ca64f11c76 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -245,14 +245,10 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
       consumers += TestUtils.createNewConsumer(TestUtils.getBrokerListStrFromServers(servers), groupId = group, securityProtocol = SecurityProtocol.PLAINTEXT)
 
     // create the consumer offset topic
-    TestUtils.createTopic(zkClient, GROUP_METADATA_TOPIC_NAME,
-      1,
-      1,
-      servers,
-      servers.head.groupCoordinator.offsetsTopicConfigs)
+    createTopic(GROUP_METADATA_TOPIC_NAME, topicConfig = servers.head.groupCoordinator.offsetsTopicConfigs)
     // create the test topic with all the brokers as replicas
-    TestUtils.createTopic(zkClient, topic, 1, 1, this.servers)
-    TestUtils.createTopic(zkClient, deleteTopic, 1, 1, this.servers)
+    createTopic(topic)
+    createTopic(deleteTopic)
   }
 
   @After
@@ -711,7 +707,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
     // create an unmatched topic
     val unmatchedTopic = "unmatched"
-    TestUtils.createTopic(zkClient, unmatchedTopic, 1, 1, this.servers)
+    createTopic(unmatchedTopic)
     addAndVerifyAcls(Set(new Acl(KafkaPrincipal.ANONYMOUS, Allow, Acl.WildCardHost, Write)),  new Resource(Topic, unmatchedTopic))
     sendRecords(1, new TopicPartition(unmatchedTopic, part))
     removeAllAcls()
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 58d1be9ee5ce5..53b3ed679bb0c 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -173,7 +173,7 @@ class ConsumerBounceTest extends IntegrationTestHarness with Logging {
     val consumer = this.consumers.head
     consumer.subscribe(Collections.singleton(newtopic))
     executor.schedule(new Runnable {
-        def run() = TestUtils.createTopic(zkClient, newtopic, serverCount, serverCount, servers)
+        def run() = createTopic(newtopic, numPartitions = serverCount, replicationFactor = serverCount)
       }, 2, TimeUnit.SECONDS)
     consumer.poll(0)
 
diff --git a/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala b/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala
index b9fb657a85356..8ca5163af7fd8 100644
--- a/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala
@@ -88,7 +88,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   @Test
   def testTooLargeRecordWithAckZero() {
     // create topic
-    TestUtils.createTopic(zkClient, topic1, 1, numServers, servers)
+    createTopic(topic1, replicationFactor = numServers)
 
     // send a too-large record
     val record = new ProducerRecord(topic1, null, "key".getBytes, new Array[Byte](serverMessageMaxBytes + 1))
@@ -105,7 +105,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   @Test
   def testTooLargeRecordWithAckOne() {
     // create topic
-    TestUtils.createTopic(zkClient, topic1, 1, numServers, servers)
+    createTopic(topic1, replicationFactor = numServers)
 
     // send a too-large record
     val record = new ProducerRecord(topic1, null, "key".getBytes, new Array[Byte](serverMessageMaxBytes + 1))
@@ -122,7 +122,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
 
     // create topic
     val topic10 = "topic10"
-    TestUtils.createTopic(zkClient, topic10, servers.size, numServers, servers, topicConfig)
+    createTopic(topic10, numPartitions = servers.size, replicationFactor = numServers, topicConfig)
 
     // send a record that is too large for replication, but within the broker max message limit
     val value = new Array[Byte](maxMessageSize - DefaultRecordBatch.RECORD_BATCH_OVERHEAD - DefaultRecord.MAX_RECORD_OVERHEAD)
@@ -169,7 +169,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   @Test
   def testWrongBrokerList() {
     // create topic
-    TestUtils.createTopic(zkClient, topic1, 1, numServers, servers)
+    createTopic(topic1, replicationFactor = numServers)
 
     // producer with incorrect broker list
     producer4 = TestUtils.createNewProducer("localhost:8686,localhost:4242", acks = 1, maxBlockMs = 10000L, bufferSize = producerBufferSize)
@@ -188,7 +188,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   @Test
   def testInvalidPartition() {
     // create topic with a single partition
-    TestUtils.createTopic(zkClient, topic1, 1, numServers, servers)
+    createTopic(topic1, numPartitions = 1, replicationFactor = numServers)
 
     // create a record with incorrect partition id (higher than the number of partitions), send should fail
     val higherRecord = new ProducerRecord(topic1, 1, "key".getBytes, "value".getBytes)
@@ -203,7 +203,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
   @Test
   def testSendAfterClosed() {
     // create topic
-    TestUtils.createTopic(zkClient, topic1, 1, numServers, servers)
+    createTopic(topic1, replicationFactor = numServers)
 
     val record = new ProducerRecord[Array[Byte],Array[Byte]](topic1, null, "key".getBytes, "value".getBytes)
 
@@ -241,7 +241,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
     val topicProps = new Properties()
     topicProps.put("min.insync.replicas",(numServers+1).toString)
 
-    TestUtils.createTopic(zkClient, topicName, 1, numServers, servers, topicProps)
+    createTopic(topicName, replicationFactor = numServers, topicConfig = topicProps)
 
     val record = new ProducerRecord(topicName, null, "key".getBytes, "value".getBytes)
     try {
@@ -261,7 +261,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness {
     val topicProps = new Properties()
     topicProps.put("min.insync.replicas", numServers.toString)
 
-    TestUtils.createTopic(zkClient, topicName, 1, numServers, servers,topicProps)
+    createTopic(topicName, replicationFactor = numServers, topicConfig = topicProps)
 
     val record = new ProducerRecord(topicName, null, "key".getBytes, "value".getBytes)
     // this should work with all brokers up and running
diff --git a/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala
index 99ddcc32493ee..5789d1ae8ca06 100644
--- a/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala
@@ -16,8 +16,9 @@ import java.io.File
 import java.util.Locale
 
 import kafka.server.KafkaConfig
-import kafka.utils.{JaasTestUtils, TestUtils}
+import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils, ZkUtils}
 import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.security.JaasUtils
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.junit.{After, Before, Test}
 
@@ -51,6 +52,8 @@ class SaslPlainPlaintextConsumerTest extends BaseConsumerTest with SaslSetup {
    */
   @Test
   def testZkAclsDisabled() {
+    val zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
     TestUtils.verifyUnsecureZkAcls(zkUtils)
+    CoreUtils.swallow(zkUtils.close(), this)
   }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala
index c28de3e9fe6ad..08351aa0c7a3d 100644
--- a/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala
@@ -16,8 +16,9 @@
   */
 package kafka.api
 
-import kafka.utils.{JaasTestUtils, TestUtils}
+import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils, ZkUtils}
 import org.apache.kafka.common.config.internals.BrokerSecurityConfigs
+import org.apache.kafka.common.security.JaasUtils
 import org.apache.kafka.common.security.auth.{AuthenticationContext, KafkaPrincipal, KafkaPrincipalBuilder, SaslAuthenticationContext}
 import org.junit.Test
 
@@ -56,6 +57,8 @@ class SaslPlainSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTes
    */
   @Test
   def testAcls() {
+    val zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
     TestUtils.verifySecureZkAcls(zkUtils, 1)
+    CoreUtils.swallow(zkUtils.close(), this)
   }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
index a4f3e0b7fa4d2..f07f4b4c49c55 100644
--- a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
@@ -19,6 +19,8 @@ package kafka.api
 import org.apache.kafka.common.security.scram.ScramMechanism
 import kafka.utils.JaasTestUtils
 import kafka.utils.ZkUtils
+import kafka.zk.ConfigEntityChangeNotificationZNode
+
 import scala.collection.JavaConverters._
 import org.junit.Before
 
@@ -32,7 +34,7 @@ class SaslScramSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTes
 
   override def configureSecurityBeforeServersStart() {
     super.configureSecurityBeforeServersStart()
-    zkClient.makeSurePersistentPathExists(ZkUtils.ConfigChangesPath)
+    zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path)
     // Create broker credentials before starting brokers
     createScramCredentials(zkConnect, kafkaPrincipal, kafkaPassword)
   }
diff --git a/core/src/test/scala/unit/kafka/admin/AdminTest.scala b/core/src/test/scala/unit/kafka/admin/AdminTest.scala
index 3b5f888f31aab..e0e26a8d566c7 100755
--- a/core/src/test/scala/unit/kafka/admin/AdminTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AdminTest.scala
@@ -22,12 +22,12 @@ import org.apache.kafka.common.errors.{InvalidReplicaAssignmentException, Invali
 import org.apache.kafka.common.metrics.Quota
 import org.easymock.EasyMock
 import org.junit.Assert._
-import org.junit.{After, Test}
+import org.junit.{After, Before, Test}
 import java.util.Properties
 
 import kafka.utils._
 import kafka.log._
-import kafka.zk.ZooKeeperTestHarness
+import kafka.zk.{ConfigEntityZNode, PreferredReplicaElectionZNode, ZooKeeperTestHarness}
 import kafka.utils.{Logging, TestUtils, ZkUtils}
 import kafka.server.{ConfigType, KafkaConfig, KafkaServer}
 import java.io.File
@@ -39,6 +39,7 @@ import kafka.utils.TestUtils._
 import scala.collection.{Map, Set, immutable}
 import kafka.utils.CoreUtils._
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.security.JaasUtils
 
 import scala.collection.JavaConverters._
 import scala.util.Try
@@ -46,9 +47,18 @@ import scala.util.Try
 class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
 
   var servers: Seq[KafkaServer] = Seq()
+  var zkUtils: ZkUtils = null
+
+  @Before
+  override def setUp() {
+    super.setUp()
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
+  }
 
   @After
   override def tearDown() {
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
     TestUtils.shutdownServers(servers)
     super.tearDown()
   }
@@ -212,9 +222,9 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
       "Partition reassignment should complete")
     val assignedReplicas = zkClient.getReplicasForPartition(new TopicPartition(topic, partitionToBeReassigned))
     // in sync replicas should not have any replica that is not in the new assigned replicas
-    checkForPhantomInSyncReplicas(zkUtils, topic, partitionToBeReassigned, assignedReplicas)
+    checkForPhantomInSyncReplicas(zkClient, topic, partitionToBeReassigned, assignedReplicas)
     assertEquals("Partition should have been reassigned to 0, 2, 3", newReplicas, assignedReplicas)
-    ensureNoUnderReplicatedPartitions(zkUtils, topic, partitionToBeReassigned, assignedReplicas, servers)
+    ensureNoUnderReplicatedPartitions(zkClient, topic, partitionToBeReassigned, assignedReplicas, servers)
     TestUtils.waitUntilTrue(() => getBrokersWithPartitionDir(servers, topic, 0) == newReplicas.toSet,
                             "New replicas should exist on brokers")
   }
@@ -242,8 +252,8 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
       "Partition reassignment should complete")
     val assignedReplicas = zkClient.getReplicasForPartition(new TopicPartition(topic, partitionToBeReassigned))
     assertEquals("Partition should have been reassigned to 0, 2, 3", newReplicas, assignedReplicas)
-    checkForPhantomInSyncReplicas(zkUtils, topic, partitionToBeReassigned, assignedReplicas)
-    ensureNoUnderReplicatedPartitions(zkUtils, topic, partitionToBeReassigned, assignedReplicas, servers)
+    checkForPhantomInSyncReplicas(zkClient, topic, partitionToBeReassigned, assignedReplicas)
+    ensureNoUnderReplicatedPartitions(zkClient, topic, partitionToBeReassigned, assignedReplicas, servers)
     TestUtils.waitUntilTrue(() => getBrokersWithPartitionDir(servers, topic, 0) == newReplicas.toSet,
                             "New replicas should exist on brokers")
   }
@@ -271,8 +281,8 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
       "Partition reassignment should complete")
     val assignedReplicas = zkClient.getReplicasForPartition(new TopicPartition(topic, partitionToBeReassigned))
     assertEquals("Partition should have been reassigned to 2, 3", newReplicas, assignedReplicas)
-    checkForPhantomInSyncReplicas(zkUtils, topic, partitionToBeReassigned, assignedReplicas)
-    ensureNoUnderReplicatedPartitions(zkUtils, topic, partitionToBeReassigned, assignedReplicas, servers)
+    checkForPhantomInSyncReplicas(zkClient, topic, partitionToBeReassigned, assignedReplicas)
+    ensureNoUnderReplicatedPartitions(zkClient, topic, partitionToBeReassigned, assignedReplicas, servers)
     TestUtils.waitUntilTrue(() => getBrokersWithPartitionDir(servers, topic, 0) == newReplicas.toSet,
                             "New replicas should exist on brokers")
   }
@@ -313,9 +323,9 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
                             "Partition reassignment should complete")
     val assignedReplicas = zkClient.getReplicasForPartition(new TopicPartition(topic, partitionToBeReassigned))
     assertEquals("Partition should have been reassigned to 0, 1", newReplicas, assignedReplicas)
-    checkForPhantomInSyncReplicas(zkUtils, topic, partitionToBeReassigned, assignedReplicas)
+    checkForPhantomInSyncReplicas(zkClient, topic, partitionToBeReassigned, assignedReplicas)
     // ensure that there are no under replicated partitions
-    ensureNoUnderReplicatedPartitions(zkUtils, topic, partitionToBeReassigned, assignedReplicas, servers)
+    ensureNoUnderReplicatedPartitions(zkClient, topic, partitionToBeReassigned, assignedReplicas, servers)
     TestUtils.waitUntilTrue(() => getBrokersWithPartitionDir(servers, topic, 0) == newReplicas.toSet,
                             "New replicas should exist on brokers")
   }
@@ -326,7 +336,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
     val partitionsForPreferredReplicaElection = Set(new TopicPartition("test", 1), new TopicPartition("test2", 1))
     PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkClient, partitionsForPreferredReplicaElection)
     // try to read it back and compare with what was written
-    val preferredReplicaElectionZkData = zkUtils.readData(ZkUtils.PreferredReplicaLeaderElectionPath)._1
+    val preferredReplicaElectionZkData = zkUtils.readData(PreferredReplicaElectionZNode.path)._1
     val partitionsUndergoingPreferredReplicaElection =
       PreferredReplicaLeaderElectionCommand.parsePreferredReplicaElectionData(preferredReplicaElectionZkData)
     assertEquals("Preferred replica election ser-de failed", partitionsForPreferredReplicaElection,
@@ -531,7 +541,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest {
     // Write config without notification to ZK.
     val configMap = Map[String, String] ("producer_byte_rate" -> "1000", "consumer_byte_rate" -> "2000")
     val map = Map("version" -> 1, "config" -> configMap.asJava)
-    zkUtils.updatePersistentPath(ZkUtils.getEntityConfigPath(ConfigType.Client, clientId), Json.encodeAsString(map.asJava))
+    zkUtils.updatePersistentPath(ConfigEntityZNode.path(ConfigType.Client, clientId), Json.encodeAsString(map.asJava))
 
     val configInZk: Map[String, Properties] = AdminUtils.fetchAllEntityConfigs(zkUtils, ConfigType.Client)
     assertEquals("Must have 1 overriden client config", 1, configInZk.size)
diff --git a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupTest.scala b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupTest.scala
index 8a731cfa8f0ad..da17f168f2df9 100644
--- a/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DeleteConsumerGroupTest.scala
@@ -20,15 +20,31 @@ import java.nio.charset.StandardCharsets
 
 import kafka.utils._
 import kafka.server.KafkaConfig
-import org.junit.Test
+import org.junit.{After, Before, Test}
 import kafka.consumer._
 import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
 import kafka.integration.KafkaServerTestHarness
+import org.apache.kafka.common.security.JaasUtils
 
 
 @deprecated("This test has been deprecated and will be removed in a future release.", "0.11.0.0")
 class DeleteConsumerGroupTest extends KafkaServerTestHarness {
   def generateConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false, true).map(KafkaConfig.fromProps)
+  var zkUtils: ZkUtils = null
+
+  @Before
+  override def setUp() {
+    super.setUp()
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
+  }
+
+  @After
+  override def tearDown() {
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
+    super.tearDown()
+  }
+
 
   @Test
   def testGroupWideDeleteInZK() {
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
index 291082ca29922..6a276df1e7c00 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
@@ -80,9 +80,9 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
     // delete the NormalTopic
     val deleteOpts = new TopicCommandOptions(Array("--topic", normalTopic))
     val deletePath = getDeleteTopicPath(normalTopic)
-    assertFalse("Delete path for topic shouldn't exist before deletion.", zkUtils.pathExists(deletePath))
+    assertFalse("Delete path for topic shouldn't exist before deletion.", zkClient.pathExists(deletePath))
     TopicCommand.deleteTopic(zkClient, deleteOpts)
-    assertTrue("Delete path for topic should exist after deletion.", zkUtils.pathExists(deletePath))
+    assertTrue("Delete path for topic should exist after deletion.", zkClient.pathExists(deletePath))
 
     // create the offset topic
     val createOffsetTopicOpts = new TopicCommandOptions(Array("--partitions", numPartitionsOriginal.toString,
@@ -93,11 +93,11 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
     // try to delete the Topic.GROUP_METADATA_TOPIC_NAME and make sure it doesn't
     val deleteOffsetTopicOpts = new TopicCommandOptions(Array("--topic", Topic.GROUP_METADATA_TOPIC_NAME))
     val deleteOffsetTopicPath = getDeleteTopicPath(Topic.GROUP_METADATA_TOPIC_NAME)
-    assertFalse("Delete path for topic shouldn't exist before deletion.", zkUtils.pathExists(deleteOffsetTopicPath))
+    assertFalse("Delete path for topic shouldn't exist before deletion.", zkClient.pathExists(deleteOffsetTopicPath))
     intercept[AdminOperationException] {
       TopicCommand.deleteTopic(zkClient, deleteOffsetTopicOpts)
     }
-    assertFalse("Delete path for topic shouldn't exist after deletion.", zkUtils.pathExists(deleteOffsetTopicPath))
+    assertFalse("Delete path for topic shouldn't exist after deletion.", zkClient.pathExists(deleteOffsetTopicPath))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala b/core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala
index 77930e6cf3714..b4381a48a85a3 100644
--- a/core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala
+++ b/core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala
@@ -28,8 +28,9 @@ import kafka.serializer._
 import kafka.server._
 import kafka.utils.TestUtils._
 import kafka.utils._
+import org.apache.kafka.common.security.JaasUtils
 import org.apache.log4j.{Level, Logger}
-import org.junit.{Test, After, Before}
+import org.junit.{After, Before, Test}
 
 import scala.collection._
 
@@ -43,6 +44,7 @@ class ZookeeperConsumerConnectorTest extends KafkaServerTestHarness with Logging
   val topic = "topic1"
   val overridingProps = new Properties()
   overridingProps.put(KafkaConfig.NumPartitionsProp, numParts.toString)
+  var zkUtils: ZkUtils = null
 
   override def generateConfigs =
     TestUtils.createBrokerConfigs(numNodes, zkConnect).map(KafkaConfig.fromProps(_, overridingProps))
@@ -57,11 +59,14 @@ class ZookeeperConsumerConnectorTest extends KafkaServerTestHarness with Logging
   @Before
   override def setUp() {
     super.setUp()
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
     dirs = new ZKGroupTopicDirs(group, topic)
   }
 
   @After
   override def tearDown() {
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
     super.tearDown()
   }
 
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 0ad64c5e21afd..88fe82bab0ef9 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -20,12 +20,12 @@ package kafka.controller
 import com.yammer.metrics.Metrics
 import com.yammer.metrics.core.Timer
 import kafka.api.LeaderAndIsr
-import kafka.common.TopicAndPartition
 import kafka.server.{KafkaConfig, KafkaServer}
-import kafka.utils.{TestUtils, ZkUtils}
-import kafka.zk.ZooKeeperTestHarness
+import kafka.utils.TestUtils
+import kafka.zk.{PreferredReplicaElectionZNode, ZooKeeperTestHarness}
 import org.junit.{After, Before, Test}
 import org.junit.Assert.assertTrue
+import org.apache.kafka.common.TopicPartition
 
 import scala.collection.JavaConverters._
 
@@ -47,37 +47,37 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
   @Test
   def testEmptyCluster(): Unit = {
     servers = makeServers(1)
-    TestUtils.waitUntilTrue(() => zkUtils.pathExists(ZkUtils.ControllerPath), "failed to elect a controller")
+    TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
     waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "broker failed to set controller epoch")
   }
 
   @Test
   def testControllerEpochPersistsWhenAllBrokersDown(): Unit = {
     servers = makeServers(1)
-    TestUtils.waitUntilTrue(() => zkUtils.pathExists(ZkUtils.ControllerPath), "failed to elect a controller")
+    TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
     waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "broker failed to set controller epoch")
     servers.head.shutdown()
     servers.head.awaitShutdown()
-    TestUtils.waitUntilTrue(() => !zkUtils.pathExists(ZkUtils.ControllerPath), "failed to kill controller")
+    TestUtils.waitUntilTrue(() => !zkClient.getControllerId.isDefined, "failed to kill controller")
     waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "controller epoch was not persisted after broker failure")
   }
 
   @Test
   def testControllerMoveIncrementsControllerEpoch(): Unit = {
     servers = makeServers(1)
-    TestUtils.waitUntilTrue(() => zkUtils.pathExists(ZkUtils.ControllerPath), "failed to elect a controller")
+    TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
     waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "broker failed to set controller epoch")
     servers.head.shutdown()
     servers.head.awaitShutdown()
     servers.head.startup()
-    TestUtils.waitUntilTrue(() => zkUtils.pathExists(ZkUtils.ControllerPath), "failed to elect a controller")
+    TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
     waitUntilControllerEpoch(KafkaController.InitialControllerEpoch + 1, "controller epoch was not incremented after controller move")
   }
 
   @Test
   def testTopicCreation(): Unit = {
     servers = makeServers(1)
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(0))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
@@ -91,7 +91,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId, controllerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers.take(1))
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch,
@@ -101,12 +101,12 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
   @Test
   def testTopicPartitionExpansion(): Unit = {
     servers = makeServers(1)
-    val tp0 = TopicAndPartition("t", 0)
-    val tp1 = TopicAndPartition("t", 1)
+    val tp0 = new TopicPartition("t", 0)
+    val tp1 = new TopicPartition("t", 1)
     val assignment = Map(tp0.partition -> Seq(0))
-    val expandedAssignment = Map(tp0.partition -> Seq(0), tp1.partition -> Seq(0))
+    val expandedAssignment = Map(tp0 -> Seq(0), tp1 -> Seq(0))
     TestUtils.createTopic(zkClient, tp0.topic, partitionReplicaAssignment = assignment, servers = servers)
-    zkUtils.updatePersistentPath(ZkUtils.getTopicPath(tp0.topic), zkUtils.replicaAssignmentZkData(expandedAssignment.map(kv => kv._1.toString -> kv._2)))
+    zkClient.setTopicAssignment(tp0.topic, expandedAssignment)
     waitForPartitionState(tp1, KafkaController.InitialControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic partition expansion")
     TestUtils.waitUntilMetadataIsPropagated(servers, tp1.topic, tp1.partition)
@@ -117,14 +117,14 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp0 = TopicAndPartition("t", 0)
-    val tp1 = TopicAndPartition("t", 1)
+    val tp0 = new TopicPartition("t", 0)
+    val tp1 = new TopicPartition("t", 1)
     val assignment = Map(tp0.partition -> Seq(otherBrokerId, controllerId))
-    val expandedAssignment = Map(tp0.partition -> Seq(otherBrokerId, controllerId), tp1.partition -> Seq(otherBrokerId, controllerId))
+    val expandedAssignment = Map(tp0 -> Seq(otherBrokerId, controllerId), tp1 -> Seq(otherBrokerId, controllerId))
     TestUtils.createTopic(zkClient, tp0.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkUtils.updatePersistentPath(ZkUtils.getTopicPath(tp0.topic), zkUtils.replicaAssignmentZkData(expandedAssignment.map(kv => kv._1.toString -> kv._2)))
+    zkClient.setTopicAssignment(tp0.topic, expandedAssignment)
     waitForPartitionState(tp1, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic partition expansion")
     TestUtils.waitUntilMetadataIsPropagated(Seq(servers(controllerId)), tp1.topic, tp1.partition)
@@ -139,16 +139,16 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val timerCount = timer(metricName).count
 
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(controllerId))
     val reassignment = Map(tp -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
-    zkUtils.createPersistentPath(ZkUtils.ReassignPartitionsPath, ZkUtils.formatAsReassignmentJson(reassignment))
+    zkClient.createPartitionReassignment(reassignment)
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 3,
       "failed to get expected partition state after partition reassignment")
-    TestUtils.waitUntilTrue(() => zkUtils.getReplicaAssignmentForTopics(Seq(tp.topic)) == reassignment,
+    TestUtils.waitUntilTrue(() =>  zkClient.getReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
       "failed to get updated partition assignment on topic znode after partition reassignment")
-    TestUtils.waitUntilTrue(() => !zkUtils.pathExists(ZkUtils.ReassignPartitionsPath),
+    TestUtils.waitUntilTrue(() => !zkClient.reassignPartitionsInProgress(),
       "failed to remove reassign partitions path after completion")
 
     val updatedTimerCount = timer(metricName).count
@@ -160,16 +160,16 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(controllerId))
     val reassignment = Map(tp -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkUtils.createPersistentPath(ZkUtils.ReassignPartitionsPath, ZkUtils.formatAsReassignmentJson(reassignment))
+    zkClient.setOrCreatePartitionReassignment(reassignment)
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state during partition reassignment with offline replica")
-    TestUtils.waitUntilTrue(() => zkUtils.pathExists(ZkUtils.ReassignPartitionsPath),
+    TestUtils.waitUntilTrue(() => zkClient.reassignPartitionsInProgress(),
       "partition reassignment path should remain while reassignment in progress")
   }
 
@@ -178,21 +178,21 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(controllerId))
     val reassignment = Map(tp -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkUtils.createPersistentPath(ZkUtils.ReassignPartitionsPath, ZkUtils.formatAsReassignmentJson(reassignment))
+    zkClient.createPartitionReassignment(reassignment)
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state during partition reassignment with offline replica")
     servers(otherBrokerId).startup()
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 4,
       "failed to get expected partition state after partition reassignment")
-    TestUtils.waitUntilTrue(() => zkUtils.getReplicaAssignmentForTopics(Seq(tp.topic)) == reassignment,
+    TestUtils.waitUntilTrue(() => zkClient.getReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
       "failed to get updated partition assignment on topic znode after partition reassignment")
-    TestUtils.waitUntilTrue(() => !zkUtils.pathExists(ZkUtils.ReassignPartitionsPath),
+    TestUtils.waitUntilTrue(() => !zkClient.reassignPartitionsInProgress(),
       "failed to remove reassign partitions path after completion")
   }
 
@@ -201,7 +201,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBroker = servers.find(_.config.brokerId != controllerId).get
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBroker.config.brokerId, controllerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     preferredReplicaLeaderElection(controllerId, otherBroker, tp, assignment(tp.partition).toSet, LeaderAndIsr.initialLeaderEpoch)
@@ -212,7 +212,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBroker = servers.find(_.config.brokerId != controllerId).get
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBroker.config.brokerId, controllerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     preferredReplicaLeaderElection(controllerId, otherBroker, tp, assignment(tp.partition).toSet, LeaderAndIsr.initialLeaderEpoch)
@@ -224,13 +224,13 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId, controllerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkUtils.createPersistentPath(ZkUtils.PreferredReplicaLeaderElectionPath, ZkUtils.preferredReplicaLeaderElectionZkData(Set(tp)))
-    TestUtils.waitUntilTrue(() => !zkUtils.pathExists(ZkUtils.PreferredReplicaLeaderElectionPath),
+    zkClient.createPreferredReplicaElection(Set(tp))
+    TestUtils.waitUntilTrue(() => !zkClient.pathExists(PreferredReplicaElectionZNode.path),
       "failed to remove preferred replica leader election path after giving up")
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state upon broker shutdown")
@@ -241,7 +241,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2, autoLeaderRebalanceEnable = true)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(1, 0))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
@@ -258,7 +258,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch,
@@ -266,7 +266,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
     TestUtils.waitUntilTrue(() => {
-      val leaderIsrAndControllerEpochMap = zkUtils.getPartitionLeaderAndIsrForTopics(Set(tp))
+      val leaderIsrAndControllerEpochMap = zkClient.getTopicPartitionStates(Seq(tp))
       leaderIsrAndControllerEpochMap.contains(tp) &&
         isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), KafkaController.InitialControllerEpoch, LeaderAndIsr.NoLeader, LeaderAndIsr.initialLeaderEpoch + 1) &&
         leaderIsrAndControllerEpochMap(tp).leaderAndIsr.isr == List(otherBrokerId)
@@ -278,7 +278,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers = makeServers(2, uncleanLeaderElectionEnable = true)
     val controllerId = TestUtils.waitUntilControllerElected(zkClient)
     val otherBrokerId = servers.map(_.config.brokerId).filter(_ != controllerId).head
-    val tp = TopicAndPartition("t", 0)
+    val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch,
@@ -286,39 +286,39 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers(1).shutdown()
     servers(1).awaitShutdown()
     TestUtils.waitUntilTrue(() => {
-      val leaderIsrAndControllerEpochMap = zkUtils.getPartitionLeaderAndIsrForTopics(Set(tp))
+      val leaderIsrAndControllerEpochMap = zkClient.getTopicPartitionStates(Seq(tp))
       leaderIsrAndControllerEpochMap.contains(tp) &&
         isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), KafkaController.InitialControllerEpoch, LeaderAndIsr.NoLeader, LeaderAndIsr.initialLeaderEpoch + 1) &&
         leaderIsrAndControllerEpochMap(tp).leaderAndIsr.isr == List(otherBrokerId)
     }, "failed to get expected partition state after entire isr went offline")
   }
 
-  private def preferredReplicaLeaderElection(controllerId: Int, otherBroker: KafkaServer, tp: TopicAndPartition,
+  private def preferredReplicaLeaderElection(controllerId: Int, otherBroker: KafkaServer, tp: TopicPartition,
                                              replicas: Set[Int], leaderEpoch: Int): Unit = {
     otherBroker.shutdown()
     otherBroker.awaitShutdown()
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, leaderEpoch + 1,
       "failed to get expected partition state upon broker shutdown")
     otherBroker.startup()
-    TestUtils.waitUntilTrue(() => zkUtils.getInSyncReplicasForPartition(tp.topic, tp.partition).toSet == replicas, "restarted broker failed to join in-sync replicas")
-    zkUtils.createPersistentPath(ZkUtils.PreferredReplicaLeaderElectionPath, ZkUtils.preferredReplicaLeaderElectionZkData(Set(tp)))
-    TestUtils.waitUntilTrue(() => !zkUtils.pathExists(ZkUtils.PreferredReplicaLeaderElectionPath),
+    TestUtils.waitUntilTrue(() => zkClient.getInSyncReplicasForPartition(new TopicPartition(tp.topic, tp.partition)).get.toSet == replicas, "restarted broker failed to join in-sync replicas")
+    zkClient.createPreferredReplicaElection(Set(tp))
+    TestUtils.waitUntilTrue(() => !zkClient.pathExists(PreferredReplicaElectionZNode.path),
       "failed to remove preferred replica leader election path after completion")
     waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBroker.config.brokerId, leaderEpoch + 2,
       "failed to get expected partition state upon broker startup")
   }
 
   private def waitUntilControllerEpoch(epoch: Int, message: String): Unit = {
-    TestUtils.waitUntilTrue(() => zkUtils.readDataMaybeNull(ZkUtils.ControllerEpochPath)._1.map(_.toInt) == Some(epoch), message)
+    TestUtils.waitUntilTrue(() => zkClient.getControllerEpoch.get._1 == epoch, message)
   }
 
-  private def waitForPartitionState(tp: TopicAndPartition,
+  private def waitForPartitionState(tp: TopicPartition,
                                     controllerEpoch: Int,
                                     leader: Int,
                                     leaderEpoch: Int,
                                     message: String): Unit = {
     TestUtils.waitUntilTrue(() => {
-      val leaderIsrAndControllerEpochMap = zkUtils.getPartitionLeaderAndIsrForTopics(Set(tp))
+      val leaderIsrAndControllerEpochMap = zkClient.getTopicPartitionStates(Seq(tp))
       leaderIsrAndControllerEpochMap.contains(tp) &&
         isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), controllerEpoch, leader, leaderEpoch)
     }, message)
diff --git a/core/src/test/scala/unit/kafka/integration/FetcherTest.scala b/core/src/test/scala/unit/kafka/integration/FetcherTest.scala
index 617b32620b44b..0a8c49f14ac24 100644
--- a/core/src/test/scala/unit/kafka/integration/FetcherTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/FetcherTest.scala
@@ -19,15 +19,16 @@ package kafka.integration
 
 import java.util.concurrent._
 import java.util.concurrent.atomic._
-import org.junit.{Test, After, Before}
+
+import org.junit.{After, Before, Test}
 
 import scala.collection._
 import org.junit.Assert._
-
 import kafka.cluster._
 import kafka.server._
 import kafka.consumer._
-import kafka.utils.TestUtils
+import kafka.utils.{CoreUtils, TestUtils, ZkUtils}
+import org.apache.kafka.common.security.JaasUtils
 
 @deprecated("This test has been deprecated and will be removed in a future release.", "0.11.0.0")
 class FetcherTest extends KafkaServerTestHarness {
@@ -39,10 +40,13 @@ class FetcherTest extends KafkaServerTestHarness {
   val queue = new LinkedBlockingQueue[FetchedDataChunk]
 
   var fetcher: ConsumerFetcherManager = null
+  var zkUtils: ZkUtils = null
 
   @Before
   override def setUp() {
     super.setUp
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
+
     createTopic(topic, partitionReplicaAssignment = Map(0 -> Seq(configs.head.brokerId)))
 
     val cluster = new Cluster(servers.map { s =>
@@ -65,6 +69,8 @@ class FetcherTest extends KafkaServerTestHarness {
   @After
   override def tearDown() {
     fetcher.stopConnections()
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
     super.tearDown
   }
 
diff --git a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
index 033ca67143b3d..19fa19dafbc27 100644
--- a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
+++ b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
@@ -18,20 +18,22 @@
 package kafka.security.auth
 
 import kafka.admin.ZkSecurityMigrator
-import kafka.utils.{Logging, TestUtils, ZkUtils}
+import kafka.utils.{CoreUtils, Logging, TestUtils, ZkUtils}
 import kafka.zk.ZooKeeperTestHarness
 import org.apache.kafka.common.KafkaException
 import org.apache.kafka.common.security.JaasUtils
-import org.apache.zookeeper.data.{ACL}
+import org.apache.zookeeper.data.ACL
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
+
 import scala.collection.JavaConverters._
-import scala.util.{Try, Success, Failure}
+import scala.util.{Failure, Success, Try}
 import javax.security.auth.login.Configuration
 
 class ZkAuthorizationTest extends ZooKeeperTestHarness with Logging {
   val jaasFile = kafka.utils.JaasTestUtils.writeJaasContextsToFile(kafka.utils.JaasTestUtils.zkSections)
   val authProvider = "zookeeper.authProvider.1"
+  var zkUtils: ZkUtils = null
 
   @Before
   override def setUp() {
@@ -39,10 +41,13 @@ class ZkAuthorizationTest extends ZooKeeperTestHarness with Logging {
     Configuration.setConfiguration(null)
     System.setProperty(authProvider, "org.apache.zookeeper.server.auth.SASLAuthenticationProvider")
     super.setUp()
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
   }
 
   @After
   override def tearDown() {
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
     super.tearDown()
     System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM)
     System.clearProperty(authProvider)
diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
index 6768b84e1bfe7..751053d3dcec3 100755
--- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
@@ -74,7 +74,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
     // create topic with 1 partition, 2 replicas, one on each broker
     val leader1 = createTopic(zkClient, topic, partitionReplicaAssignment = Map(0 -> Seq(0, 1)), servers = servers)(0)
 
-    val leaderEpoch1 = zkUtils.getEpochForPartition(topic, partitionId)
+    val leaderEpoch1 = zkClient.getEpochForPartition(new TopicPartition(topic, partitionId)).get
     debug("leader Epoch: " + leaderEpoch1)
     debug("Leader is elected to be: %s".format(leader1))
     // NOTE: this is to avoid transient test failures
@@ -86,7 +86,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
     // check if leader moves to the other server
     val leader2 = waitUntilLeaderIsElectedOrChanged(zkClient, topic, partitionId,
                                                     oldLeaderOpt = if (leader1 == 0) None else Some(leader1))
-    val leaderEpoch2 = zkUtils.getEpochForPartition(topic, partitionId)
+    val leaderEpoch2 = zkClient.getEpochForPartition(new TopicPartition(topic, partitionId)).get
     debug("Leader is elected to be: %s".format(leader1))
     debug("leader Epoch: " + leaderEpoch2)
     assertEquals("Leader must move to broker 0", 0, leader2)
@@ -100,7 +100,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
     Thread.sleep(zookeeper.tickTime)
     val leader3 = waitUntilLeaderIsElectedOrChanged(zkClient, topic, partitionId,
                                                     oldLeaderOpt = if (leader2 == 1) None else Some(leader2))
-    val leaderEpoch3 = zkUtils.getEpochForPartition(topic, partitionId)
+    val leaderEpoch3 = zkClient.getEpochForPartition(new TopicPartition(topic, partitionId)).get
     debug("leader Epoch: " + leaderEpoch3)
     debug("Leader is elected to be: %s".format(leader3))
     assertEquals("Leader must return to 1", 1, leader3)
@@ -119,7 +119,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
     // create topic with 1 partition, 2 replicas, one on each broker
     val leader1 = createTopic(zkClient, topic, partitionReplicaAssignment = Map(0 -> Seq(0, 1)), servers = servers)(0)
 
-    val leaderEpoch1 = zkUtils.getEpochForPartition(topic, partitionId)
+    val leaderEpoch1 = zkClient.getEpochForPartition(new TopicPartition(topic, partitionId)).get
     debug("leader Epoch: " + leaderEpoch1)
     debug("Leader is elected to be: %s".format(leader1))
     // NOTE: this is to avoid transient test failures
diff --git a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
index ba33ab051b573..2087363bfbd7a 100644
--- a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala
@@ -24,7 +24,7 @@ import kafka.server.LogDirFailureTest._
 import kafka.api.IntegrationTestHarness
 import kafka.controller.{OfflineReplica, PartitionAndReplica}
 import kafka.utils.{CoreUtils, Exit, TestUtils}
-import kafka.zk.LogDirEventNotificationZNode
+
 import org.apache.kafka.clients.consumer.KafkaConsumer
 import org.apache.kafka.clients.producer.{ProducerConfig, ProducerRecord}
 import org.apache.kafka.common.TopicPartition
@@ -191,7 +191,7 @@ class LogDirFailureTest extends IntegrationTestHarness {
     }, "Expected some messages", 3000L)
 
     // There should be no remaining LogDirEventNotification znode
-    assertTrue(zkUtils.getChildrenParentMayNotExist(LogDirEventNotificationZNode.path).isEmpty)
+    assertTrue(zkClient.getAllLogDirEventNotifications.isEmpty)
 
     // The controller should have marked the replica on the original leader as offline
     val controllerServer = servers.find(_.kafkaController.isActive).get
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
index 29a1fa6a6b6da..2fa66009739e2 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateBrokerIdTest.scala
@@ -189,12 +189,4 @@ class ServerGenerateBrokerIdTest extends ZooKeeperTestHarness {
     }
     true
   }
-
-  @Test
-  def testGetSequenceIdMethod() {
-    val path = "/test/seqid"
-    (1 to 10).foreach { seqid =>
-      assertEquals(seqid, zkUtils.getSequenceId(path))
-    }
-  }
 }
diff --git a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
index 54ba8876e343b..0317da309252c 100755
--- a/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerGenerateClusterIdTest.scala
@@ -49,7 +49,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
   @Test
   def testAutoGenerateClusterId() {
     // Make sure that the cluster id doesn't exist yet.
-    assertFalse(zkUtils.pathExists(ZkUtils.ClusterIdPath))
+    assertFalse(zkClient.getClusterId.isDefined)
 
     var server1 = TestUtils.createServer(config1)
     servers = Seq(server1)
@@ -61,7 +61,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     server1.shutdown()
 
     // Make sure that the cluster id is persistent.
-    assertTrue(zkUtils.pathExists(ZkUtils.ClusterIdPath))
+    assertTrue(zkClient.getClusterId.isDefined)
     assertEquals(zkClient.getClusterId, Some(clusterIdOnFirstBoot))
 
     // Restart the server check to confirm that it uses the clusterId generated previously
@@ -74,7 +74,7 @@ class ServerGenerateClusterIdTest extends ZooKeeperTestHarness {
     server1.shutdown()
 
     // Make sure that the cluster id is persistent after multiple reboots.
-    assertTrue(zkUtils.pathExists(ZkUtils.ClusterIdPath))
+    assertTrue(zkClient.getClusterId.isDefined)
     assertEquals(zkClient.getClusterId, Some(clusterIdOnFirstBoot))
 
     TestUtils.verifyNonDaemonThreadsStatus(this.getClass.getName)
diff --git a/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala b/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala
index 3414c36ecfe26..4c05d982dd796 100755
--- a/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala
@@ -45,7 +45,7 @@ class ServerStartupTest extends ZooKeeperTestHarness {
     props.put("zookeeper.connect", zooKeeperConnect + zookeeperChroot)
     server = TestUtils.createServer(KafkaConfig.fromProps(props))
 
-    val pathExists = zkUtils.pathExists(zookeeperChroot)
+    val pathExists = zkClient.pathExists(zookeeperChroot)
     assertTrue(pathExists)
   }
 
@@ -76,7 +76,7 @@ class ServerStartupTest extends ZooKeeperTestHarness {
     val brokerId = 0
     val props1 = TestUtils.createBrokerConfig(brokerId, zkConnect)
     server = TestUtils.createServer(KafkaConfig.fromProps(props1))
-    val brokerRegistration = zkUtils.readData(ZkUtils.BrokerIdsPath + "/" + brokerId)._1
+    val brokerRegistration = zkClient.getBroker(brokerId).getOrElse(fail("broker doesn't exists"))
 
     val props2 = TestUtils.createBrokerConfig(brokerId, zkConnect)
     try {
@@ -88,7 +88,7 @@ class ServerStartupTest extends ZooKeeperTestHarness {
     }
 
     // broker registration shouldn't change
-    assertEquals(brokerRegistration, zkUtils.readData(ZkUtils.BrokerIdsPath + "/" + brokerId)._1)
+    assertEquals(brokerRegistration, zkClient.getBroker(brokerId).getOrElse(fail("broker doesn't exists")))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala b/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
index 3a42c3ba55335..3389161e78b3e 100644
--- a/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
@@ -19,12 +19,12 @@ package kafka.utils
 
 import kafka.server.{KafkaConfig, ReplicaFetcherManager}
 import kafka.api.LeaderAndIsr
-import kafka.zk.ZooKeeperTestHarness
+import kafka.controller.LeaderIsrAndControllerEpoch
+import kafka.zk.{IsrChangeNotificationZNode, TopicZNode, ZooKeeperTestHarness}
 import org.apache.kafka.common.TopicPartition
 import org.junit.Assert._
 import org.junit.{Before, Test}
 import org.easymock.EasyMock
-import scala.collection.JavaConverters._
 
 class ReplicationUtilsTest extends ZooKeeperTestHarness {
   private val zkVersion = 1
@@ -34,14 +34,15 @@ class ReplicationUtilsTest extends ZooKeeperTestHarness {
   private val leaderEpoch = 1
   private val controllerEpoch = 1
   private val isr = List(1, 2)
-  private val topicPath = s"/brokers/topics/$topic/partitions/$partition/state"
-  private val topicData = Json.encodeAsString(Map("controller_epoch" -> controllerEpoch, "leader" -> leader,
-    "versions" -> 1, "leader_epoch" -> leaderEpoch, "isr" -> isr).asJava)
 
   @Before
   override def setUp() {
     super.setUp()
-    zkUtils.createPersistentPath(topicPath, topicData)
+    zkClient.makeSurePersistentPathExists(TopicZNode.path(topic))
+    val topicPartition = new TopicPartition(topic, partition)
+    val leaderAndIsr = LeaderAndIsr(leader, leaderEpoch, isr, 1)
+    val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)
+    zkClient.createTopicPartitionStatesRaw(Map(topicPartition -> leaderIsrAndControllerEpoch))
   }
 
   @Test
@@ -63,7 +64,7 @@ class ReplicationUtilsTest extends ZooKeeperTestHarness {
     EasyMock.expect(replicaManager.zkClient).andReturn(zkClient)
     EasyMock.replay(replicaManager)
 
-    zkClient.makeSurePersistentPathExists(ZkUtils.IsrChangeNotificationPath)
+    zkClient.makeSurePersistentPathExists(IsrChangeNotificationZNode.path)
 
     val replicas = List(0, 1)
 
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 303afa720aef8..636c0bd7337c5 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -977,24 +977,25 @@ object TestUtils extends Logging {
     file.close()
   }
 
-  def checkForPhantomInSyncReplicas(zkUtils: ZkUtils, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int]) {
-    val inSyncReplicas = zkUtils.getInSyncReplicasForPartition(topic, partitionToBeReassigned)
+  def checkForPhantomInSyncReplicas(zkClient: KafkaZkClient, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int]) {
+    val inSyncReplicas = zkClient.getInSyncReplicasForPartition(new TopicPartition(topic, partitionToBeReassigned))
     // in sync replicas should not have any replica that is not in the new assigned replicas
-    val phantomInSyncReplicas = inSyncReplicas.toSet -- assignedReplicas.toSet
+    val phantomInSyncReplicas = inSyncReplicas.get.toSet -- assignedReplicas.toSet
     assertTrue("All in sync replicas %s must be in the assigned replica list %s".format(inSyncReplicas, assignedReplicas),
       phantomInSyncReplicas.isEmpty)
   }
 
-  def ensureNoUnderReplicatedPartitions(zkUtils: ZkUtils, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int],
+  def ensureNoUnderReplicatedPartitions(zkClient: KafkaZkClient, topic: String, partitionToBeReassigned: Int, assignedReplicas: Seq[Int],
                                                 servers: Seq[KafkaServer]) {
+    val topicPartition = new TopicPartition(topic, partitionToBeReassigned)
     TestUtils.waitUntilTrue(() => {
-        val inSyncReplicas = zkUtils.getInSyncReplicasForPartition(topic, partitionToBeReassigned)
-        inSyncReplicas.size == assignedReplicas.size
+        val inSyncReplicas = zkClient.getInSyncReplicasForPartition(topicPartition)
+        inSyncReplicas.get.size == assignedReplicas.size
       },
       "Reassigned partition [%s,%d] is under replicated".format(topic, partitionToBeReassigned))
     var leader: Option[Int] = None
     TestUtils.waitUntilTrue(() => {
-        leader = zkUtils.getLeaderForPartition(topic, partitionToBeReassigned)
+        leader = zkClient.getLeaderForPartition(topicPartition)
         leader.isDefined
       },
       "Reassigned partition [%s,%d] is unavailable".format(topic, partitionToBeReassigned))
diff --git a/core/src/test/scala/unit/kafka/utils/ZkUtilsTest.scala b/core/src/test/scala/unit/kafka/utils/ZkUtilsTest.scala
index 9f78124d03149..292db8b88958e 100755
--- a/core/src/test/scala/unit/kafka/utils/ZkUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/ZkUtilsTest.scala
@@ -21,12 +21,27 @@ import kafka.api.LeaderAndIsr
 import kafka.common.TopicAndPartition
 import kafka.controller.LeaderIsrAndControllerEpoch
 import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.common.security.JaasUtils
 import org.junit.Assert._
-import org.junit.Test
+import org.junit.{After, Before, Test}
 
 class ZkUtilsTest extends ZooKeeperTestHarness {
 
   val path = "/path"
+  var zkUtils: ZkUtils = _
+
+  @Before
+  override def setUp() {
+    super.setUp
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
+  }
+
+  @After
+  override def tearDown() {
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
+    super.tearDown
+  }
 
   @Test
   def testSuccessfulConditionalDeletePath() {
@@ -107,4 +122,11 @@ class ZkUtilsTest extends ZooKeeperTestHarness {
     assertEquals(None, zkUtils.getLeaderIsrAndEpochForPartition(topic, partition + 1))
   }
 
+  @Test
+  def testGetSequenceIdMethod() {
+    val path = "/test/seqid"
+    (1 to 10).foreach { seqid =>
+      assertEquals(seqid, zkUtils.getSequenceId(path))
+    }
+  }
 }
diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala
index 6280d97b23ce5..06ea963077214 100644
--- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala
@@ -21,18 +21,15 @@ import java.lang.Iterable
 import javax.security.auth.login.Configuration
 
 import scala.collection.JavaConverters._
-
 import kafka.consumer.ConsumerConfig
-import kafka.utils.ZkUtils
-import kafka.utils.ZKCheckedEphemeral
-import kafka.utils.TestUtils
+import kafka.utils.{CoreUtils, TestUtils, ZKCheckedEphemeral, ZkUtils}
 import org.apache.kafka.common.security.JaasUtils
 import org.apache.zookeeper.CreateMode
 import org.apache.zookeeper.WatchedEvent
 import org.apache.zookeeper.Watcher
 import org.apache.zookeeper.ZooDefs.Ids
 import org.I0Itec.zkclient.exception.ZkNodeExistsException
-import org.junit.{After, Before, Test, Assert}
+import org.junit.{After, Assert, Before, Test}
 import org.junit.runners.Parameterized
 import org.junit.runners.Parameterized.Parameters
 import org.junit.runner.RunWith
@@ -50,7 +47,8 @@ class ZKEphemeralTest(val secure: Boolean) extends ZooKeeperTestHarness {
   val jaasFile = kafka.utils.JaasTestUtils.writeJaasContextsToFile(kafka.utils.JaasTestUtils.zkSections)
   val authProvider = "zookeeper.authProvider.1"
   var zkSessionTimeoutMs = 1000
-  
+  var zkUtils: ZkUtils = null
+
   @Before
   override def setUp() {
     if (secure) {
@@ -61,10 +59,13 @@ class ZKEphemeralTest(val secure: Boolean) extends ZooKeeperTestHarness {
         fail("Secure access not enabled")
     }
     super.setUp
+    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
   }
   
   @After
   override def tearDown() {
+    if (zkUtils != null)
+     CoreUtils.swallow(zkUtils.close(), this)
     super.tearDown
     System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM)
     System.clearProperty(authProvider)
diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
index c7c3152ca482e..9e7258315d860 100755
--- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
+++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
@@ -19,7 +19,7 @@ package kafka.zk
 
 import javax.security.auth.login.Configuration
 
-import kafka.utils.{CoreUtils, Logging, TestUtils, ZkUtils}
+import kafka.utils.{CoreUtils, Logging, TestUtils}
 import org.junit.{After, AfterClass, Before, BeforeClass}
 import org.junit.Assert._
 import org.scalatest.junit.JUnitSuite
@@ -43,7 +43,6 @@ abstract class ZooKeeperTestHarness extends JUnitSuite with Logging {
 
   protected val zkAclsEnabled: Option[Boolean] = None
 
-  var zkUtils: ZkUtils = null
   var zkClient: KafkaZkClient = null
   var adminZkClient: AdminZkClient = null
 
@@ -55,7 +54,6 @@ abstract class ZooKeeperTestHarness extends JUnitSuite with Logging {
   @Before
   def setUp() {
     zookeeper = new EmbeddedZookeeper()
-    zkUtils = ZkUtils(zkConnect, zkSessionTimeout, zkConnectionTimeout, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled))
     zkClient = KafkaZkClient(zkConnect, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled), zkSessionTimeout,
       zkConnectionTimeout, zkMaxInFlightRequests, Time.SYSTEM)
     adminZkClient = new AdminZkClient(zkClient)
@@ -63,8 +61,6 @@ abstract class ZooKeeperTestHarness extends JUnitSuite with Logging {
 
   @After
   def tearDown() {
-    if (zkUtils != null)
-     CoreUtils.swallow(zkUtils.close(), this)
     if (zkClient != null)
      zkClient.close()
     if (zookeeper != null)

From 1067cc3422e6f2731ae57df9f01e84f2242a60be Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 15 Feb 2018 17:36:44 +0000
Subject: [PATCH 0060/1847] KAFKA-6512: Discard references to buffers used for
 compression (#4570)

ProducerBatch retains references to MemoryRecordsBuilder and cannot be freed until acks are received. Removing references to buffers used for compression after records are built will enable these to be garbage collected sooner, reducing the risk of OOM.

Reviewers: Ismael Juma , Jason Gustafson , Lothsahn 
---
 .../record/KafkaLZ4BlockOutputStream.java     | 40 ++++++++++++-------
 .../common/record/MemoryRecordsBuilder.java   | 21 ++++++----
 .../record/MemoryRecordsBuilderTest.java      | 34 ++++++++++++++++
 3 files changed, 73 insertions(+), 22 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockOutputStream.java b/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockOutputStream.java
index 8cfc37be826cb..591ab1693646c 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockOutputStream.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockOutputStream.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.record;
 
-import java.io.FilterOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 
@@ -34,7 +33,7 @@
  *
  * This class is not thread-safe.
  */
-public final class KafkaLZ4BlockOutputStream extends FilterOutputStream {
+public final class KafkaLZ4BlockOutputStream extends OutputStream {
 
     public static final int MAGIC = 0x184D2204;
     public static final int LZ4_MAX_HEADER_LENGTH = 19;
@@ -52,9 +51,10 @@ public final class KafkaLZ4BlockOutputStream extends FilterOutputStream {
     private final boolean useBrokenFlagDescriptorChecksum;
     private final FLG flg;
     private final BD bd;
-    private final byte[] buffer;
-    private final byte[] compressedBuffer;
     private final int maxBlockSize;
+    private OutputStream out;
+    private byte[] buffer;
+    private byte[] compressedBuffer;
     private int bufferOffset;
     private boolean finished;
 
@@ -71,7 +71,7 @@ public final class KafkaLZ4BlockOutputStream extends FilterOutputStream {
      * @throws IOException
      */
     public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum, boolean useBrokenFlagDescriptorChecksum) throws IOException {
-        super(out);
+        this.out = out;
         compressor = LZ4Factory.fastestInstance().fastCompressor();
         checksum = XXHashFactory.fastestInstance().hash32();
         this.useBrokenFlagDescriptorChecksum = useBrokenFlagDescriptorChecksum;
@@ -204,7 +204,6 @@ private void writeBlock() throws IOException {
     private void writeEndMark() throws IOException {
         ByteUtils.writeUnsignedIntLE(out, 0);
         // TODO implement content checksum, update flg.validate()
-        finished = true;
     }
 
     @Override
@@ -259,15 +258,26 @@ private void ensureNotFinished() {
 
     @Override
     public void close() throws IOException {
-        if (!finished) {
-            // basically flush the buffer writing the last block
-            writeBlock();
-            // write the end block and finish the stream
-            writeEndMark();
-        }
-        if (out != null) {
-            out.close();
-            out = null;
+        try {
+            if (!finished) {
+                // basically flush the buffer writing the last block
+                writeBlock();
+                // write the end block
+                writeEndMark();
+            }
+        } finally {
+            try {
+                if (out != null) {
+                    try (OutputStream outStream = out) {
+                        outStream.flush();
+                    }
+                }
+            } finally {
+                out = null;
+                buffer = null;
+                compressedBuffer = null;
+                finished = true;
+            }
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
index a9b57ac22df09..6f6404fa2d959 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
@@ -23,6 +23,7 @@
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 
 import static org.apache.kafka.common.utils.Utils.wrapNullable;
@@ -38,11 +39,15 @@
  */
 public class MemoryRecordsBuilder {
     private static final float COMPRESSION_RATE_ESTIMATION_FACTOR = 1.05f;
+    private static final DataOutputStream CLOSED_STREAM = new DataOutputStream(new OutputStream() {
+        @Override
+        public void write(int b) throws IOException {
+            throw new IllegalStateException("MemoryRecordsBuilder is closed for record appends");
+        }
+    });
 
     private final TimestampType timestampType;
     private final CompressionType compressionType;
-    // Used to append records, may compress data on the fly
-    private final DataOutputStream appendStream;
     // Used to hold a reference to the underlying ByteBuffer so that we can write the record batch header and access
     // the written bytes. ByteBufferOutputStream allocates a new ByteBuffer if the existing one is not large enough,
     // so it's not safe to hold a direct reference to the underlying ByteBuffer.
@@ -60,7 +65,8 @@ public class MemoryRecordsBuilder {
     // from previous batches before appending any records.
     private float estimatedCompressionRatio = 1.0F;
 
-    private boolean appendStreamIsClosed = false;
+    // Used to append records, may compress data on the fly
+    private DataOutputStream appendStream;
     private boolean isTransactional;
     private long producerId;
     private short producerEpoch;
@@ -265,12 +271,13 @@ public void overrideLastOffset(long lastOffset) {
      * possible to update the RecordBatch header.
      */
     public void closeForRecordAppends() {
-        if (!appendStreamIsClosed) {
+        if (appendStream != CLOSED_STREAM) {
             try {
                 appendStream.close();
-                appendStreamIsClosed = true;
             } catch (IOException e) {
                 throw new KafkaException(e);
+            } finally {
+                appendStream = CLOSED_STREAM;
             }
         }
     }
@@ -663,7 +670,7 @@ private void recordWritten(long offset, long timestamp, int size) {
     }
 
     private void ensureOpenForRecordAppend() {
-        if (appendStreamIsClosed)
+        if (appendStream == CLOSED_STREAM)
             throw new IllegalStateException("Tried to append a record, but MemoryRecordsBuilder is closed for record appends");
     }
 
@@ -738,7 +745,7 @@ public boolean isClosed() {
     public boolean isFull() {
         // note that the write limit is respected only after the first record is added which ensures we can always
         // create non-empty batches (this is used to disable batching when the producer's batch size is set to 0).
-        return appendStreamIsClosed || (this.numRecords > 0 && this.writeLimit <= estimatedBytesWritten());
+        return appendStream == CLOSED_STREAM || (this.numRecords > 0 && this.writeLimit <= estimatedBytesWritten());
     }
 
     /**
diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
index c713d17f61545..a90fb299c52b2 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
@@ -28,6 +28,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -644,6 +645,39 @@ public static Collection data() {
         return values;
     }
 
+    @Test
+    public void testBuffersDereferencedOnClose() {
+        Runtime runtime = Runtime.getRuntime();
+        int payloadLen = 1024 * 1024;
+        ByteBuffer buffer = ByteBuffer.allocate(payloadLen * 2);
+        byte[] key = new byte[0];
+        byte[] value = new byte[payloadLen];
+        new Random().nextBytes(value); // Use random payload so that compressed buffer is large
+        List builders = new ArrayList<>(100);
+        long startMem = 0;
+        long memUsed = 0;
+        int iterations =  0;
+        while (iterations++ < 100) {
+            buffer.rewind();
+            MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V2, compressionType,
+                    TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID,
+                    RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false,
+                    RecordBatch.NO_PARTITION_LEADER_EPOCH, 0);
+            builder.append(1L, new byte[0], value);
+            builder.build();
+            builders.add(builder);
+
+            System.gc();
+            memUsed = runtime.totalMemory() - runtime.freeMemory() - startMem;
+            // Ignore memory usage during initialization
+            if (iterations == 2)
+                startMem = memUsed;
+            else if (iterations > 2 && memUsed < (iterations - 2) * 1024)
+                break;
+        }
+        assertTrue("Memory usage too high: " + memUsed, iterations < 100);
+    }
+
     private void verifyRecordsProcessingStats(RecordsProcessingStats processingStats, int numRecords,
             int numRecordsConverted, long finalBytes, long preConvertedBytes) {
         assertNotNull("Records processing info is null", processingStats);

From bce8125dcf5a87ed29bbe795cd7591d45bdf3d9a Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Thu, 15 Feb 2018 10:52:58 -0800
Subject: [PATCH 0061/1847] MINOR: Resuming Tasks should not be initialized
 twice (#4562)

Avoids double initialization of resuming tasks
Removes race condition in StreamThreadTest plus code cleanup

Author: Matthias J. Sax 

Reviewers: Bill Bejeck , Guozhang Wang 
---
 .../processor/internals/StreamTask.java       |  1 -
 .../internals/StreamsMetricsImpl.java         |  6 +--
 .../processor/internals/StreamThreadTest.java | 37 ++++++++++---------
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 56c0ab31f4b5a..6bca02ad9bc8e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -192,7 +192,6 @@ public void resume() {
             }
             transactionInFlight = true;
         }
-        initTopology();
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImpl.java
index 03bbceb25c485..b2ce2e7dcf827 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImpl.java
@@ -129,11 +129,11 @@ public Sensor addLatencyAndThroughputSensor(String scopeName, String entityName,
 
         // first add the global operation metrics if not yet, with the global tags only
         Sensor parent = metrics.sensor(sensorName(operationName, null), recordingLevel);
-        addLatencyMetrics(scopeName, parent, operationName, allTagMap);
+        addLatencyAndThroughputMetrics(scopeName, parent, operationName, allTagMap);
 
         // add the operation metrics with additional tags
         Sensor sensor = metrics.sensor(sensorName(operationName, entityName), recordingLevel, parent);
-        addLatencyMetrics(scopeName, sensor, operationName, tagMap);
+        addLatencyAndThroughputMetrics(scopeName, sensor, operationName, tagMap);
 
         parentSensors.put(sensor, parent);
 
@@ -161,7 +161,7 @@ public Sensor addThroughputSensor(String scopeName, String entityName, String op
         return sensor;
     }
 
-    private void addLatencyMetrics(String scopeName, Sensor sensor, String opName, Map tags) {
+    private void addLatencyAndThroughputMetrics(String scopeName, Sensor sensor, String opName, Map tags) {
 
         maybeAddMetric(sensor, metrics.metricName(opName + "-latency-avg", groupNameFromScope(scopeName),
             "The average latency of " + opName + " operation.", tags), new Avg());
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index cc056044792b0..482b764f3ff58 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -133,8 +133,6 @@ private Properties configProps(final boolean enableEos) {
         };
     }
 
-
-    @SuppressWarnings("unchecked")
     @Test
     public void testPartitionAssignmentChangeForSingleGroup() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
@@ -170,14 +168,13 @@ public void testPartitionAssignmentChangeForSingleGroup() {
         assertTrue(thread.state() == StreamThread.State.PENDING_SHUTDOWN);
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void testStateChangeStartClose() throws InterruptedException {
-
         final StreamThread thread = createStreamThread(clientId, config, false);
 
         final StateListenerStub stateListener = new StateListenerStub();
         thread.setStateListener(stateListener);
+
         thread.start();
         TestUtils.waitForCondition(new TestCondition() {
             @Override
@@ -185,19 +182,20 @@ public boolean conditionMet() {
                 return thread.state() == StreamThread.State.RUNNING;
             }
         }, 10 * 1000, "Thread never started.");
+
         thread.shutdown();
-        assertEquals(thread.state(), StreamThread.State.PENDING_SHUTDOWN);
         TestUtils.waitForCondition(new TestCondition() {
             @Override
             public boolean conditionMet() {
                 return thread.state() == StreamThread.State.DEAD;
             }
         }, 10 * 1000, "Thread never shut down.");
+
         thread.shutdown();
         assertEquals(thread.state(), StreamThread.State.DEAD);
     }
 
-    private Cluster createCluster(int numNodes) {
+    private Cluster createCluster(final int numNodes) {
         HashMap nodes = new HashMap<>();
         for (int i = 0; i < numNodes; ++i) {
             nodes.put(i, new Node(i, "localhost", 8121 + i));
@@ -362,7 +360,7 @@ private TaskManager mockTaskManagerCommit(final Consumer consume
     }
 
     @Test
-    public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEosDisabled() throws InterruptedException {
+    public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEosDisabled() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
         final StreamThread thread = createStreamThread(clientId, config, false);
@@ -394,7 +392,7 @@ public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEo
     }
 
     @Test
-    public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosEnable() throws InterruptedException {
+    public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosEnable() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
         final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
@@ -423,7 +421,7 @@ public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosEnable()
     }
 
     @Test
-    public void shouldCloseAllTaskProducersOnCloseIfEosEnabled() throws InterruptedException {
+    public void shouldCloseAllTaskProducersOnCloseIfEosEnabled() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
         final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
@@ -455,7 +453,7 @@ public void shouldCloseAllTaskProducersOnCloseIfEosEnabled() throws InterruptedE
 
     @SuppressWarnings("unchecked")
     @Test
-    public void shouldShutdownTaskManagerOnClose() throws InterruptedException {
+    public void shouldShutdownTaskManagerOnClose() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
         final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
         EasyMock.expect(taskManager.activeTasks()).andReturn(Collections.emptyMap());
@@ -493,6 +491,7 @@ public void onChange(final Thread t, final ThreadStateTransitionValidator newSta
         EasyMock.verify(taskManager);
     }
 
+    @SuppressWarnings("unchecked")
     @Test
     public void shouldShutdownTaskManagerOnCloseWithoutStart() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
@@ -521,6 +520,7 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() {
         EasyMock.verify(taskManager);
     }
 
+    @SuppressWarnings("unchecked")
     @Test
     public void shouldOnlyShutdownOnce() {
         final Consumer consumer = EasyMock.createNiceMock(Consumer.class);
@@ -552,7 +552,7 @@ public void shouldOnlyShutdownOnce() {
     }
 
     @Test
-    public void shouldNotNullPointerWhenStandbyTasksAssignedAndNoStateStoresForTopology() throws InterruptedException {
+    public void shouldNotNullPointerWhenStandbyTasksAssignedAndNoStateStoresForTopology() {
         internalTopologyBuilder.addSource(null, "name", null, null, null, "topic");
         internalTopologyBuilder.addSink("out", "output", null, null, null);
 
@@ -573,7 +573,7 @@ public void shouldNotNullPointerWhenStandbyTasksAssignedAndNoStateStoresForTopol
     }
 
     @Test
-    public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWhileProcessing() throws InterruptedException {
+    public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWhileProcessing() throws Exception {
         internalTopologyBuilder.addSource(null, "source", null, null, null, topic1);
         internalTopologyBuilder.addSink("sink", "dummyTopic", null, null, null, "source");
 
@@ -682,7 +682,8 @@ private static class StateListenerStub implements StreamThread.StateListener {
         ThreadStateTransitionValidator newState = null;
 
         @Override
-        public void onChange(final Thread thread, final ThreadStateTransitionValidator newState,
+        public void onChange(final Thread thread,
+                             final ThreadStateTransitionValidator newState,
                              final ThreadStateTransitionValidator oldState) {
             ++numChanges;
             if (this.newState != null) {
@@ -696,7 +697,7 @@ public void onChange(final Thread thread, final ThreadStateTransitionValidator n
     }
 
     @Test
-    public void shouldReturnActiveTaskMetadataWhileRunningState() throws InterruptedException {
+    public void shouldReturnActiveTaskMetadataWhileRunningState() {
         internalTopologyBuilder.addSource(null, "source", null, null, null, topic1);
 
         final StreamThread thread = createStreamThread(clientId, config, false);
@@ -726,7 +727,7 @@ public void shouldReturnActiveTaskMetadataWhileRunningState() throws Interrupted
     }
 
     @Test
-    public void shouldReturnStandbyTaskMetadataWhileRunningState() throws InterruptedException {
+    public void shouldReturnStandbyTaskMetadataWhileRunningState() {
         internalStreamsBuilder.stream(Collections.singleton(topic1), consumed)
             .groupByKey().count(Materialized.>as("count-one"));
 
@@ -911,7 +912,7 @@ public void close() { }
     }
 
     @Test
-    public void shouldAlwaysUpdateTasksMetadataAfterChangingState() throws InterruptedException {
+    public void shouldAlwaysUpdateTasksMetadataAfterChangingState() {
         final StreamThread thread = createStreamThread(clientId, config, false);
         ThreadMetadata metadata = thread.threadMetadata();
         assertEquals(StreamThread.State.CREATED.name(), metadata.threadState());
@@ -922,7 +923,7 @@ public void shouldAlwaysUpdateTasksMetadataAfterChangingState() throws Interrupt
     }
 
     @Test
-    public void shouldAlwaysReturnEmptyTasksMetadataWhileRebalancingStateAndTasksNotRunning() throws InterruptedException {
+    public void shouldAlwaysReturnEmptyTasksMetadataWhileRebalancingStateAndTasksNotRunning() {
         internalStreamsBuilder.stream(Collections.singleton(topic1), consumed)
             .groupByKey().count(Materialized.>as("count-one"));
 
@@ -1063,7 +1064,7 @@ public boolean conditionMet() {
     }
 
     @Test
-    public void shouldReportSkippedRecordsForInvalidTimestamps() throws Exception {
+    public void shouldReportSkippedRecordsForInvalidTimestamps() {
         internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
 
         final Properties config = configProps(false);

From c72062ddc089c338a3017aff6db702ca74e56847 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 15 Feb 2018 20:21:12 +0000
Subject: [PATCH 0062/1847] KAFKA-6517: Avoid deadlock in ZooKeeperClient
 during session expiry (#4551)

ZooKeeperClient acquires initializationLock#writeLock to establish a new connection while processing session expiry WatchEvent. ZooKeeperClient#handleRequests acquires initializationLock#readLock, allowing multiple batches of requests to be processed concurrently, but preventing reconnections while processing requests. At the moment, handleRequests holds onto the readLock throughout the method, even while waiting for responses and inflight requests to complete. But responses cannot be delivered if event thread is blocked on the writeLock to process session expiry event. This results in a deadlock. During broker shutdown, the shutdown thread is also blocked since it needs the readLock to perform ZooKeeperClient#unregisterStateChangeHandler, which cannot be acquired if a session expiry had occurred earlier since this thread gets queued behind the event handler thread waiting for writeLock.

This commit reduces locking in ZooKeeperClient#handleRequests to just the non-blocking send, so that session expiry handling doesn't get blocked when a send is blocked waiting for responses. Also moves session expiry handling to a separate thread so that Kafka controller doesn't block the event handler thread when processing session expiry.
---
 .../kafka/zookeeper/ZooKeeperClient.scala     |  39 ++++--
 .../kafka/zookeeper/ZooKeeperClientTest.scala | 114 +++++++++++++++++-
 2 files changed, 137 insertions(+), 16 deletions(-)

diff --git a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
index 9a1d16274dfa3..3934fd0ad5d6b 100644
--- a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
+++ b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.{ArrayBlockingQueue, ConcurrentHashMap, CountDownLat
 import com.yammer.metrics.core.{Gauge, MetricName}
 import kafka.metrics.KafkaMetricsGroup
 import kafka.utils.CoreUtils.{inLock, inReadLock, inWriteLock}
-import kafka.utils.Logging
+import kafka.utils.{KafkaScheduler, Logging}
 import org.apache.kafka.common.utils.Time
 import org.apache.zookeeper.AsyncCallback.{ACLCallback, Children2Callback, DataCallback, StatCallback, StringCallback, VoidCallback}
 import org.apache.zookeeper.KeeperException.Code
@@ -59,6 +59,7 @@ class ZooKeeperClient(connectString: String,
   private val zNodeChildChangeHandlers = new ConcurrentHashMap[String, ZNodeChildChangeHandler]().asScala
   private val inFlightRequests = new Semaphore(maxInFlightRequests)
   private val stateChangeHandlers = new ConcurrentHashMap[String, StateChangeHandler]().asScala
+  private[zookeeper] val expiryScheduler = new KafkaScheduler(0, "zk-session-expiry-handler")
 
   private val metricNames = Set[String]()
 
@@ -90,6 +91,7 @@ class ZooKeeperClient(connectString: String,
 
   metricNames += "SessionState"
 
+  expiryScheduler.startup()
   waitUntilConnected(connectionTimeoutMs, TimeUnit.MILLISECONDS)
 
   override def metricName(name: String, metricTags: scala.collection.Map[String, String]): MetricName = {
@@ -122,7 +124,7 @@ class ZooKeeperClient(connectString: String,
    * response type (e.g. Seq[CreateRequest] -> Seq[CreateResponse]). Otherwise, the most specific common supertype
    * will be used (e.g. Seq[AsyncRequest] -> Seq[AsyncResponse]).
    */
-  def handleRequests[Req <: AsyncRequest](requests: Seq[Req]): Seq[Req#Response] = inReadLock(initializationLock) {
+  def handleRequests[Req <: AsyncRequest](requests: Seq[Req]): Seq[Req#Response] = {
     if (requests.isEmpty)
       Seq.empty
     else {
@@ -132,10 +134,12 @@ class ZooKeeperClient(connectString: String,
       requests.foreach { request =>
         inFlightRequests.acquire()
         try {
-          send(request) { response =>
-            responseQueue.add(response)
-            inFlightRequests.release()
-            countDownLatch.countDown()
+          inReadLock(initializationLock) {
+            send(request) { response =>
+              responseQueue.add(response)
+              inFlightRequests.release()
+              countDownLatch.countDown()
+            }
           }
         } catch {
           case e: Throwable =>
@@ -148,7 +152,8 @@ class ZooKeeperClient(connectString: String,
     }
   }
 
-  private def send[Req <: AsyncRequest](request: Req)(processResponse: Req#Response => Unit): Unit = {
+  // Visibility to override for testing
+  private[zookeeper] def send[Req <: AsyncRequest](request: Req)(processResponse: Req#Response => Unit): Unit = {
     // Safe to cast as we always create a response of the right type
     def callback(response: AsyncResponse): Unit = processResponse(response.asInstanceOf[Req#Response])
 
@@ -303,12 +308,18 @@ class ZooKeeperClient(connectString: String,
     stateChangeHandlers.clear()
     zooKeeper.close()
     metricNames.foreach(removeMetric(_))
+    expiryScheduler.shutdown()
     info("Closed.")
   }
 
   def sessionId: Long = inReadLock(initializationLock) {
     zooKeeper.getSessionId
   }
+
+  // Only for testing
+  private[zookeeper] def currentZooKeeper: ZooKeeper = inReadLock(initializationLock) {
+    zooKeeper
+  }
   
   private def initialize(): Unit = {
     if (!connectionState.isAlive) {
@@ -352,12 +363,14 @@ class ZooKeeperClient(connectString: String,
             error("Auth failed.")
             stateChangeHandlers.values.foreach(_.onAuthFailure())
           } else if (state == KeeperState.Expired) {
-            inWriteLock(initializationLock) {
-              info("Session expired.")
-              stateChangeHandlers.values.foreach(_.beforeInitializingSession())
-              initialize()
-              stateChangeHandlers.values.foreach(_.afterInitializingSession())
-            }
+            expiryScheduler.schedule("zk-session-expired", () => {
+              inWriteLock(initializationLock) {
+                info("Session expired.")
+                stateChangeHandlers.values.foreach(_.beforeInitializingSession())
+                initialize()
+                stateChangeHandlers.values.foreach(_.afterInitializingSession())
+              }
+            }, delay = 0L, period = -1L, unit = TimeUnit.MILLISECONDS)
           }
         case Some(path) =>
           (event.getType: @unchecked) match {
diff --git a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
index c8ebaa907355a..f1c09d7308d73 100644
--- a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
@@ -19,7 +19,7 @@ package kafka.zookeeper
 import java.nio.charset.StandardCharsets
 import java.util.UUID
 import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.{ArrayBlockingQueue, CountDownLatch, TimeUnit}
+import java.util.concurrent.{ArrayBlockingQueue, ConcurrentLinkedQueue, CountDownLatch, Executors, Semaphore, TimeUnit}
 
 import com.yammer.metrics.Metrics
 import com.yammer.metrics.core.{Gauge, Meter, MetricName}
@@ -29,8 +29,8 @@ import org.apache.kafka.common.utils.Time
 import org.apache.zookeeper.KeeperException.{Code, NoNodeException}
 import org.apache.zookeeper.Watcher.Event.{EventType, KeeperState}
 import org.apache.zookeeper.ZooKeeper.States
-import org.apache.zookeeper.{CreateMode, WatchedEvent, ZooDefs}
-import org.junit.Assert.{assertArrayEquals, assertEquals, assertTrue}
+import org.apache.zookeeper.{CreateMode, WatchedEvent, Watcher, ZooDefs, ZooKeeper}
+import org.junit.Assert.{assertArrayEquals, assertEquals, assertFalse, assertNull, assertTrue}
 import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
@@ -385,6 +385,114 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
     } finally zooKeeperClient.close()
   }
 
+  /**
+    * Tests that if session expiry notification is received while a thread is processing requests,
+    * session expiry is handled and the request thread completes with responses to all requests,
+    * even though some requests may fail due to session expiry or disconnection.
+    *
+    * Sequence of events on different threads:
+    *   Request thread:
+    *       - Sends `maxInflightRequests` requests (these may complete before session is expired)
+    *   Main thread:
+    *       - Waits for at least one request to be processed (this should succeed)
+    *       - Expires session by creating new client with same session id
+    *       - Unblocks another `maxInflightRequests` requests before and after new client is closed (these may fail)
+    *   ZooKeeperClient Event thread:
+    *       - Delivers responses and session expiry (no ordering guarantee between these, both are processed asynchronously)
+    *   Response executor thread:
+    *       - Blocks subsequent sends by delaying response until session expiry is processed
+    *   ZooKeeperClient Session Expiry Handler:
+    *       - Unblocks subsequent sends
+    *   Main thread:
+    *       - Waits for all sends to complete. The requests sent after session expiry processing should succeed.
+    */
+  @Test
+  def testSessionExpiry(): Unit = {
+    val maxInflightRequests = 2
+    val responseExecutor = Executors.newSingleThreadExecutor
+    val sendSemaphore = new Semaphore(0)
+    val sendCompleteSemaphore = new Semaphore(0)
+    val sendSize = maxInflightRequests * 5
+    @volatile var resultCodes: Seq[Code] = null
+    val stateChanges = new ConcurrentLinkedQueue[String]()
+    val zooKeeperClient = new ZooKeeperClient(zkConnect, zkSessionTimeout, zkConnectionTimeout, maxInflightRequests,
+      time, "testGroupType", "testGroupName") {
+      override def send[Req <: AsyncRequest](request: Req)(processResponse: Req#Response => Unit): Unit = {
+        super.send(request)( response => {
+          responseExecutor.submit(new Runnable {
+            override def run(): Unit = {
+              sendCompleteSemaphore.release()
+              sendSemaphore.acquire()
+              processResponse(response)
+            }
+          })
+        })
+      }
+    }
+    try {
+      zooKeeperClient.registerStateChangeHandler(new StateChangeHandler {
+        override val name: String ="test-state-change-handler"
+        override def afterInitializingSession(): Unit = {
+          verifyHandlerThread()
+          stateChanges.add("afterInitializingSession")
+        }
+        override def beforeInitializingSession(): Unit = {
+          verifyHandlerThread()
+          stateChanges.add("beforeInitializingSession")
+          sendSemaphore.release(sendSize) // Resume remaining sends
+        }
+        private def verifyHandlerThread(): Unit = {
+          val threadName = Thread.currentThread.getName
+          assertTrue(s"Unexpected thread + $threadName", threadName.startsWith(zooKeeperClient.expiryScheduler.threadNamePrefix))
+        }
+      })
+
+      val requestThread = new Thread {
+        override def run(): Unit = {
+          val requests = (1 to sendSize).map(i => GetDataRequest(s"/$i"))
+          resultCodes = zooKeeperClient.handleRequests(requests).map(_.resultCode)
+        }
+      }
+      requestThread.start()
+      sendCompleteSemaphore.acquire() // Wait for request thread to start processing requests
+
+      // Trigger session expiry by reusing the session id in another client
+      val dummyWatcher = new Watcher {
+        override def process(event: WatchedEvent): Unit = {}
+      }
+      val anotherZkClient = new ZooKeeper(zkConnect, 1000, dummyWatcher,
+        zooKeeperClient.currentZooKeeper.getSessionId,
+        zooKeeperClient.currentZooKeeper.getSessionPasswd)
+      assertNull(anotherZkClient.exists("/nonexistent", false)) // Make sure new client works
+      sendSemaphore.release(maxInflightRequests) // Resume a few more sends which may fail
+      anotherZkClient.close()
+      sendSemaphore.release(maxInflightRequests) // Resume a few more sends which may fail
+
+      requestThread.join(10000)
+      if (requestThread.isAlive) {
+        requestThread.interrupt()
+        fail("Request thread did not complete")
+      }
+      assertEquals(Seq("beforeInitializingSession", "afterInitializingSession"), stateChanges.asScala.toSeq)
+
+      assertEquals(resultCodes.size, sendSize)
+      val connectionLostCount = resultCodes.count(_ == Code.CONNECTIONLOSS)
+      assertTrue(s"Unexpected connection lost requests $resultCodes", connectionLostCount <= maxInflightRequests)
+      val expiredCount = resultCodes.count(_ == Code.SESSIONEXPIRED)
+      assertTrue(s"Unexpected session expired requests $resultCodes", expiredCount <= maxInflightRequests)
+      assertTrue(s"No connection lost or expired requests $resultCodes", connectionLostCount + expiredCount > 0)
+      assertEquals(Code.NONODE, resultCodes.head)
+      assertEquals(Code.NONODE, resultCodes.last)
+      assertTrue(s"Unexpected result code $resultCodes",
+        resultCodes.filterNot(Set(Code.NONODE, Code.SESSIONEXPIRED, Code.CONNECTIONLOSS).contains).isEmpty)
+
+    } finally {
+      zooKeeperClient.close()
+      responseExecutor.shutdownNow()
+    }
+    assertFalse("Expiry executor not shutdown", zooKeeperClient.expiryScheduler.isStarted)
+  }
+
   def isExpectedMetricName(metricName: MetricName, name: String): Boolean =
     metricName.getName == name && metricName.getGroup == "testMetricGroup" && metricName.getType == "testMetricType"
 

From 004ef6b04bbc0ce0ebfc1cad42656559b043bcb2 Mon Sep 17 00:00:00 2001
From: Eugene Sevastyanov 
Date: Fri, 16 Feb 2018 01:11:00 +0300
Subject: [PATCH 0063/1847] MINOR: Cache Node's hashCode to improve the
 producer's performance  (#4350)

`Node` is immutable so this is safe.

With 100 brokers, 150 topics and 350 partitions, `HashSet.contains` in `RecordAccumulator.ready` took about 40% of the application time. It
is caused by re-calculating a hash code of a leader (Node instance) for
every batch entry. Caching the hashCode reduced the time of
`HashSet.contains` in `RecordAccumulator.ready` to ~2%. The
measurements were taken with Flight Recorder.

Reviewers: Rajini Sivaram , Ted Yu , Ismael Juma 
---
 .../java/org/apache/kafka/common/Node.java    | 45 ++++++++-----------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/Node.java b/clients/src/main/java/org/apache/kafka/common/Node.java
index 8187369ca3c59..d51fa13946b22 100644
--- a/clients/src/main/java/org/apache/kafka/common/Node.java
+++ b/clients/src/main/java/org/apache/kafka/common/Node.java
@@ -29,12 +29,14 @@ public class Node {
     private final int port;
     private final String rack;
 
+    // Cache hashCode as it is called in performance sensitive parts of the code (e.g. RecordAccumulator.ready)
+    private Integer hash;
+
     public Node(int id, String host, int port) {
         this(id, host, port, null);
     }
 
     public Node(int id, String host, int port, String rack) {
-        super();
         this.id = id;
         this.idString = Integer.toString(id);
         this.host = host;
@@ -100,39 +102,30 @@ public String rack() {
 
     @Override
     public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + ((host == null) ? 0 : host.hashCode());
-        result = prime * result + id;
-        result = prime * result + port;
-        result = prime * result + ((rack == null) ? 0 : rack.hashCode());
-        return result;
+        Integer h = this.hash;
+        if (h == null) {
+            int result = 31 + ((host == null) ? 0 : host.hashCode());
+            result = 31 * result + id;
+            result = 31 * result + port;
+            result = 31 * result + ((rack == null) ? 0 : rack.hashCode());
+            this.hash = result;
+            return result;
+        } else {
+            return h;
+        }
     }
 
     @Override
     public boolean equals(Object obj) {
         if (this == obj)
             return true;
-        if (obj == null)
-            return false;
-        if (getClass() != obj.getClass())
+        if (obj == null || getClass() != obj.getClass())
             return false;
         Node other = (Node) obj;
-        if (host == null) {
-            if (other.host != null)
-                return false;
-        } else if (!host.equals(other.host))
-            return false;
-        if (id != other.id)
-            return false;
-        if (port != other.port)
-            return false;
-        if (rack == null) {
-            if (other.rack != null)
-                return false;
-        } else if (!rack.equals(other.rack))
-            return false;
-        return true;
+        return (host == null ? other.host == null : host.equals(other.host)) &&
+            id == other.id &&
+            port == other.port &&
+            (rack == null ? other.rack == null : rack.equals(other.rack));
     }
 
     @Override

From f12d237fdd47302d1c6c77553db986dd6dda568a Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 16 Feb 2018 07:58:49 -0800
Subject: [PATCH 0064/1847] MINOR: Free sends in MultiSend as they complete
 (#4574)

Currently we hold onto all Records references in a multi-partition fetch response until the full response has completed. This can be a problem when the records have been down-converted since they will be occupying a (potentially large) chunk of memory. This patch changes the behavior in MultiSend so that once a Send is completed, we no longer keep a reference to it, which will allow the Records objects to be freed sooner.

I have added a simple unit test to verify that sends are removed as the MultiSend progresses.

Reviewers: Ismael Juma 
---
 .../kafka/common/network/MultiSend.java       | 36 +++++----
 .../kafka/common/requests/FetchResponse.java  | 10 ++-
 .../kafka/common/network/MultiSendTest.java   | 79 +++++++++++++++++++
 3 files changed, 106 insertions(+), 19 deletions(-)
 create mode 100644 clients/src/test/java/org/apache/kafka/common/network/MultiSendTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java
index f77ff971e8039..6b66360930535 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java
@@ -22,32 +22,35 @@
 
 import java.io.IOException;
 import java.nio.channels.GatheringByteChannel;
-import java.util.Iterator;
-import java.util.List;
+import java.util.Queue;
 
 /**
  * A set of composite sends, sent one after another
  */
-
 public class MultiSend implements Send {
-
     private static final Logger log = LoggerFactory.getLogger(MultiSend.class);
 
     private final String dest;
-    private final Iterator sendsIterator;
+    private final Queue sendQueue;
     private final long size;
 
     private long totalWritten = 0;
     private Send current;
 
-    public MultiSend(String dest, List sends) {
+    /**
+     * Construct a MultiSend for the given destination from a queue of Send objects. The queue will be
+     * consumed as the MultiSend progresses (on completion, it will be empty).
+     */
+    public MultiSend(String dest, Queue sends) {
         this.dest = dest;
-        this.sendsIterator = sends.iterator();
-        nextSendOrDone();
+        this.sendQueue = sends;
+
         long size = 0;
         for (Send send : sends)
             size += send.size();
         this.size = size;
+
+        this.current = sendQueue.poll();
     }
 
     @Override
@@ -65,6 +68,15 @@ public boolean completed() {
         return current == null;
     }
 
+    // Visible for testing
+    int numResidentSends() {
+        int count = 0;
+        if (current != null)
+            count += 1;
+        count += sendQueue.size();
+        return count;
+    }
+
     @Override
     public long writeTo(GatheringByteChannel channel) throws IOException {
         if (completed())
@@ -77,7 +89,7 @@ public long writeTo(GatheringByteChannel channel) throws IOException {
             totalWrittenPerCall += written;
             sendComplete = current.completed();
             if (sendComplete)
-                nextSendOrDone();
+                current = sendQueue.poll();
         } while (!completed() && sendComplete);
 
         totalWritten += totalWrittenPerCall;
@@ -91,10 +103,4 @@ public long writeTo(GatheringByteChannel channel) throws IOException {
         return totalWrittenPerCall;
     }
 
-    private void nextSendOrDone() {
-        if (sendsIterator.hasNext())
-            current = sendsIterator.next();
-        else
-            current = null;
-    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
index 98c6be333e15b..ed0f5a347dd5f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
@@ -29,12 +29,14 @@
 import org.apache.kafka.common.record.Records;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
@@ -359,7 +361,7 @@ protected Send toSend(String dest, ResponseHeader responseHeader, short apiVersi
         responseHeaderStruct.writeTo(buffer);
         buffer.rewind();
 
-        List sends = new ArrayList<>();
+        Queue sends = new ArrayDeque<>();
         sends.add(new ByteBufferSend(dest, buffer));
         addResponseData(responseBodyStruct, throttleTimeMs, dest, sends);
         return new MultiSend(dest, sends);
@@ -393,7 +395,7 @@ public static FetchResponse parse(ByteBuffer buffer, short version) {
         return new FetchResponse(ApiKeys.FETCH.responseSchema(version).read(buffer));
     }
 
-    private static void addResponseData(Struct struct, int throttleTimeMs, String dest, List sends) {
+    private static void addResponseData(Struct struct, int throttleTimeMs, String dest, Queue sends) {
         Object[] allTopicData = struct.getArray(RESPONSES_KEY_NAME);
 
         if (struct.hasField(ERROR_CODE)) {
@@ -421,7 +423,7 @@ private static void addResponseData(Struct struct, int throttleTimeMs, String de
             addTopicData(dest, sends, (Struct) topicData);
     }
 
-    private static void addTopicData(String dest, List sends, Struct topicData) {
+    private static void addTopicData(String dest, Queue sends, Struct topicData) {
         String topic = topicData.get(TOPIC_NAME);
         Object[] allPartitionData = topicData.getArray(PARTITIONS_KEY_NAME);
 
@@ -436,7 +438,7 @@ private static void addTopicData(String dest, List sends, Struct topicData
             addPartitionData(dest, sends, (Struct) partitionData);
     }
 
-    private static void addPartitionData(String dest, List sends, Struct partitionData) {
+    private static void addPartitionData(String dest, Queue sends, Struct partitionData) {
         Struct header = partitionData.getStruct(PARTITION_HEADER_KEY_NAME);
         Records records = partitionData.getRecords(RECORD_SET_KEY_NAME);
 
diff --git a/clients/src/test/java/org/apache/kafka/common/network/MultiSendTest.java b/clients/src/test/java/org/apache/kafka/common/network/MultiSendTest.java
new file mode 100644
index 0000000000000..d2b2ef6c5d24b
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/network/MultiSendTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.network;
+
+import org.apache.kafka.test.TestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.Queue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class MultiSendTest {
+
+    @Test
+    public void testSendsFreedAfterWriting() throws IOException {
+        String dest = "1";
+        int numChunks = 4;
+        int chunkSize = 32;
+        int totalSize = numChunks * chunkSize;
+
+        Queue sends = new LinkedList<>();
+        ByteBuffer[] chunks = new ByteBuffer[numChunks];
+
+        for (int i = 0; i < numChunks; i++) {
+            ByteBuffer buffer = ByteBuffer.wrap(TestUtils.randomBytes(chunkSize));
+            chunks[i] = buffer;
+            sends.add(new ByteBufferSend(dest, buffer));
+        }
+
+        MultiSend send = new MultiSend(dest, sends);
+        assertEquals(totalSize, send.size());
+
+        for (int i = 0; i < numChunks; i++) {
+            assertEquals(numChunks - i, send.numResidentSends());
+            NonOverflowingByteBufferChannel out = new NonOverflowingByteBufferChannel(chunkSize);
+            send.writeTo(out);
+            out.close();
+            assertEquals(chunks[i], out.buffer());
+        }
+
+        assertEquals(0, send.numResidentSends());
+        assertTrue(send.completed());
+    }
+
+    private static class NonOverflowingByteBufferChannel extends org.apache.kafka.common.requests.ByteBufferChannel {
+
+        private NonOverflowingByteBufferChannel(long size) {
+            super(size);
+        }
+
+        @Override
+        public long write(ByteBuffer[] srcs) throws IOException {
+            // Instead of overflowing, this channel refuses additional writes once the buffer is full,
+            // which allows us to test the MultiSend behavior on a per-send basis.
+            if (!buffer().hasRemaining())
+                return 0;
+            return super.write(srcs);
+        }
+    }
+
+}

From 55aa7de62adf6a99e6d6268c832b61de6b37d416 Mon Sep 17 00:00:00 2001
From: dan norwood 
Date: Fri, 16 Feb 2018 08:24:04 -0800
Subject: [PATCH 0065/1847] MINOR: Make it explicit in consumer docs that
 poll() is needed for callback to run (#4480)

Make it clear in the docs that the rebalance listener is only invoked during an active call to `poll()`. Plus a few additional doc cleanups.

Reviewers: Ismael Juma , Jason Gustafson 
---
 .../consumer/ConsumerRebalanceListener.java   |  4 +--
 .../kafka/clients/consumer/KafkaConsumer.java | 36 +++++++++----------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index 845bff3fc4931..a37e1661079d3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -44,7 +44,7 @@
  * partition is reassigned it may want to automatically trigger a flush of this cache, before the new owner takes over
  * consumption.
  * 
    
- * This callback will execute in the user thread as part of the {@link Consumer#poll(long) poll(long)} call whenever partition assignment changes.
+ * This callback will only execute in the user thread as part of the {@link Consumer#poll(long) poll(long)} call whenever partition assignment changes.
  * 
    
  * It is guaranteed that all consumer processes will invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} prior to
  * any process invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}. So if offsets or other state is saved in the
@@ -103,7 +103,7 @@ public interface ConsumerRebalanceListener {
     /**
      * A callback method the user can implement to provide handling of customized offsets on completion of a successful
      * partition re-assignment. This method will be called after an offset re-assignment completes and before the
-     * consumer starts fetching data.
+     * consumer starts fetching data, and only as the result of a {@link Consumer#poll(long) poll(long)} call.
      * 
    
      * It is guaranteed that all the processes in a consumer group will execute their
      * {@link #onPartitionsRevoked(Collection)} callback before any instance executes its
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index cbe54c77b2ac8..ad96ecf0d2a50 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -868,17 +868,20 @@ public Set subscription() {
      *
      * 
    
      * As part of group management, the consumer will keep track of the list of consumers that belong to a particular
-     * group and will trigger a rebalance operation if one of the following events trigger -
+     * group and will trigger a rebalance operation if any one of the following events are triggered:
      * 
      
-     * 
    • Number of partitions change for any of the subscribed list of topics
-     * 
    • Topic is created or deleted
-     * 
    • An existing member of the consumer group dies
-     * 
    • A new member is added to an existing consumer group via the join API
+     * 
    • Number of partitions change for any of the subscribed topics
+     * 
    • A subscribed topic is created or deleted
+     * 
    • An existing member of the consumer group is shutdown or fails
+     * 
    • A new member is added to the consumer group
      * 
    
      * 
    
      * When any of these events are triggered, the provided listener will be invoked first to indicate that
      * the consumer's assignment has been revoked, and then again when the new assignment has been received.
-     * Note that this listener will immediately override any listener set in a previous call to subscribe.
+     * Note that rebalances will only occur during an active call to {@link #poll(long)}, so callbacks will
+     * also only be invoked during that time.
+     *
+     * The provided listener will immediately override any listener set in a previous call to subscribe.
      * It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics
      * subscribed in this call. See {@link ConsumerRebalanceListener} for more details.
      *
@@ -926,7 +929,7 @@ public void subscribe(Collection topics, ConsumerRebalanceListener liste
      *
      * 
    
      * This is a short-hand for {@link #subscribe(Collection, ConsumerRebalanceListener)}, which
-     * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer
+     * uses a no-op listener. If you need the ability to seek to particular offsets, you should prefer
      * {@link #subscribe(Collection, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets
      * to be reset. You should also provide your own listener if you are doing your own offset
      * management since the listener gives you an opportunity to commit offsets before a rebalance finishes.
@@ -944,17 +947,14 @@ public void subscribe(Collection topics) {
 
     /**
      * Subscribe to all topics matching specified pattern to get dynamically assigned partitions.
-     * The pattern matching will be done periodically against topic existing at the time of check.
+     * The pattern matching will be done periodically against all topics existing at the time of check.
+     * This can be controlled through the {@code metadata.max.age.ms} configuration: by lowering
+     * the max metadata age, the consumer will refresh metadata more often and check for matching topics.
      * 
    
-     * As part of group management, the consumer will keep track of the list of consumers that
-     * belong to a particular group and will trigger a rebalance operation if one of the
-     * following events trigger -
-     * 
      
-     * 
    • Number of partitions change for any of the subscribed list of topics
-     * 
    • Topic is created or deleted
-     * 
    • An existing member of the consumer group dies
-     * 
    • A new member is added to an existing consumer group via the join API
-     * 
    
+     * See {@link #subscribe(Collection, ConsumerRebalanceListener)} for details on the
+     * use of the {@link ConsumerRebalanceListener}. Generally rebalances are triggered when there
+     * is a change to the topics matching the provided pattern and when consumer group membership changes.
+     * Group rebalances only take place during an active call to {@link #poll(long)}.
      *
      * @param pattern Pattern to subscribe to
      * @param listener Non-null listener instance to get notifications on partition assignment/revocation for the
@@ -988,7 +988,7 @@ public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) {
      * The pattern matching will be done periodically against topics existing at the time of check.
      * 
    
      * This is a short-hand for {@link #subscribe(Pattern, ConsumerRebalanceListener)}, which
-     * uses a noop listener. If you need the ability to seek to particular offsets, you should prefer
+     * uses a no-op listener. If you need the ability to seek to particular offsets, you should prefer
      * {@link #subscribe(Pattern, ConsumerRebalanceListener)}, since group rebalances will cause partition offsets
      * to be reset. You should also provide your own listener if you are doing your own offset
      * management since the listener gives you an opportunity to commit offsets before a rebalance finishes.

From 4837d44673490f6adf1e62466702f49c7f411250 Mon Sep 17 00:00:00 2001
From: Attila Sasvari 
Date: Fri, 16 Feb 2018 17:30:14 +0100
Subject: [PATCH 0066/1847] MINOR: Fix typos in kafka.py (#4581)

---
 tests/kafkatest/services/kafka/kafka.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index b3d617754b692..30dcdb59c36ce 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -103,9 +103,9 @@ def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAI
         # is authenticating and waiting for the SaslAuthenticated
         # in addition to the SyncConnected event.
         #
-        # The defaut value for zookeeper.connect.timeout.ms is
+        # The default value for zookeeper.connect.timeout.ms is
         # 2 seconds and here we increase it to 5 seconds, but
-        # it can be overriden by setting the corresponding parameter
+        # it can be overridden by setting the corresponding parameter
         # for this constructor.
         self.zk_connect_timeout = zk_connect_timeout
 

From 7303f41dc171e3a8f57c40abc467582602d1cde4 Mon Sep 17 00:00:00 2001
From: Filipe Agapito 
Date: Fri, 16 Feb 2018 20:36:21 +0000
Subject: [PATCH 0067/1847] KAFKA-6424:
 QueryableStateIntegrationTest#queryOnRebalance should accept raw text (#4549)

Allows input data to be read from a file and removes .toLowerCase in word count stream

Author: Filipe Agapito 

Reviewers: Ted Yu , Matthias J. Sax 
---
 .../QueryableStateIntegrationTest.java        | 64 +++++++++++++------
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
index fc2eaccebc947..8b4e8957ed59b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
@@ -65,15 +65,20 @@
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
@@ -90,6 +95,8 @@
 
 @Category({IntegrationTest.class})
 public class QueryableStateIntegrationTest {
+    private static final Logger log = LoggerFactory.getLogger(QueryableStateIntegrationTest.class);
+
     private static final int NUM_BROKERS = 1;
 
     @ClassRule
@@ -133,6 +140,40 @@ private void createTopics() throws InterruptedException {
         CLUSTER.createTopics(outputTopic, outputTopicConcurrent, outputTopicConcurrentWindowed, outputTopicThree);
     }
 
+    /**
+     * Try to read inputValues from {@code resources/QueryableStateIntegrationTest/inputValues.txt}, which might be useful
+     * for larger scale testing. In case of exception, for instance if no such file can be read, return a small list
+     * which satisfies all the prerequisites of the tests.
+     */
+    private List getInputValues() {
+        List input = new ArrayList<>();
+        final ClassLoader classLoader = getClass().getClassLoader();
+        final String fileName = "QueryableStateIntegrationTest" + File.separator + "inputValues.txt";
+        try (final BufferedReader reader = new BufferedReader(new FileReader(classLoader.getResource(fileName).getFile()))) {
+            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+                input.add(line);
+            }
+        } catch (final Exception e) {
+            log.warn("Unable to read '{}{}{}'. Using default inputValues list", "resources", File.separator, fileName);
+            input = Arrays.asList(
+                        "hello world",
+                        "all streams lead to kafka",
+                        "streams",
+                        "kafka streams",
+                        "the cat in the hat",
+                        "green eggs and ham",
+                        "that Sam i am",
+                        "up the creek without a paddle",
+                        "run forest run",
+                        "a tank full of gas",
+                        "eat sleep rave repeat",
+                        "one jolly sailor",
+                        "king of the world");
+
+        }
+        return input;
+    }
+
     @Before
     public void before() throws Exception {
         testNo++;
@@ -167,20 +208,7 @@ public int compare(final KeyValue o1,
                 return o1.key.compareTo(o2.key);
             }
         };
-        inputValues = Arrays.asList(
-            "hello world",
-            "all streams lead to kafka",
-            "streams",
-            "kafka streams",
-            "the cat in the hat",
-            "green eggs and ham",
-            "that sam i am",
-            "up the creek without a paddle",
-            "run forest run",
-            "a tank full of gas",
-            "eat sleep rave repeat",
-            "one jolly sailor",
-            "king of the world");
+        inputValues = getInputValues();
         inputValuesKeys = new HashSet<>();
         for (final String sentence : inputValues) {
             final String[] words = sentence.split("\\W+");
@@ -215,7 +243,7 @@ private KafkaStreams createCountStream(final String inputTopic,
             .flatMapValues(new ValueMapper>() {
                 @Override
                 public Iterable apply(final String value) {
-                    return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
+                    return Arrays.asList(value.split("\\W+"));
                 }
             })
             .groupBy(MockMapper.selectValueMapper());
@@ -376,7 +404,7 @@ public void queryOnRebalance() throws InterruptedException {
                     storeName + "-" + streamThree);
                 verifyAllWindowedKeys(streamRunnables, streamRunnables[i].getStream(), streamRunnables[i].getStateListener(), inputValuesKeys,
                     windowStoreName + "-" + streamThree, 0L, WINDOW_SIZE);
-                assertEquals(streamRunnables[i].getStream().state(), KafkaStreams.State.RUNNING);
+                assertEquals(KafkaStreams.State.RUNNING, streamRunnables[i].getStream().state());
             }
 
             // kill N-1 threads
@@ -391,7 +419,7 @@ public void queryOnRebalance() throws InterruptedException {
                 storeName + "-" + streamThree);
             verifyAllWindowedKeys(streamRunnables, streamRunnables[0].getStream(), streamRunnables[0].getStateListener(), inputValuesKeys,
                 windowStoreName + "-" + streamThree, 0L, WINDOW_SIZE);
-            assertEquals(streamRunnables[0].getStream().state(), KafkaStreams.State.RUNNING);
+            assertEquals(KafkaStreams.State.RUNNING, streamRunnables[0].getStream().state());
         } finally {
             for (int i = 0; i < numThreads; i++) {
                 if (!streamRunnables[i].isClosed()) {

From 0b3b6049f002acf040a5312cef35c87f199bbfd2 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Fri, 16 Feb 2018 13:13:13 -0800
Subject: [PATCH 0068/1847] MINOR: Fix Streams EOS system tests (#4572)

Avoid loosing log/stdout/stderr files on restart
Reenables tests

Author: Matthias J. Sax 

Reviewers: Guozhang Wang , Bill Bejeck 
---
 tests/kafkatest/services/streams.py               |  8 ++++++--
 tests/kafkatest/tests/streams/streams_eos_test.py | 10 ++++------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py
index 0f484b4ca3384..9df87ca794594 100644
--- a/tests/kafkatest/services/streams.py
+++ b/tests/kafkatest/services/streams.py
@@ -154,12 +154,18 @@ def __init__(self, test_context, kafka, command):
 class StreamsEosTestBaseService(StreamsTestBaseService):
     """Base class for Streams EOS Test services providing some common settings and functionality"""
 
+    clean_node_enabled = True
+
     def __init__(self, test_context, kafka, command):
         super(StreamsEosTestBaseService, self).__init__(test_context,
                                                         kafka,
                                                         "org.apache.kafka.streams.tests.StreamsEosTest",
                                                         command)
 
+    def clean_node(self, node):
+        if self.clean_node_enabled:
+            super.clean_node(self, node)
+
 
 class StreamsSmokeTestDriverService(StreamsSmokeTestBaseService):
     def __init__(self, test_context, kafka):
@@ -180,12 +186,10 @@ class StreamsEosTestJobRunnerService(StreamsEosTestBaseService):
     def __init__(self, test_context, kafka):
         super(StreamsEosTestJobRunnerService, self).__init__(test_context, kafka, "process")
 
-
 class StreamsComplexEosTestJobRunnerService(StreamsEosTestBaseService):
     def __init__(self, test_context, kafka):
         super(StreamsComplexEosTestJobRunnerService, self).__init__(test_context, kafka, "process-complex")
 
-
 class StreamsEosTestVerifyRunnerService(StreamsEosTestBaseService):
     def __init__(self, test_context, kafka):
         super(StreamsEosTestVerifyRunnerService, self).__init__(test_context, kafka, "verify")
diff --git a/tests/kafkatest/tests/streams/streams_eos_test.py b/tests/kafkatest/tests/streams/streams_eos_test.py
index 27002e937da5a..d6ac600d2ab14 100644
--- a/tests/kafkatest/tests/streams/streams_eos_test.py
+++ b/tests/kafkatest/tests/streams/streams_eos_test.py
@@ -20,7 +20,6 @@
 from kafkatest.services.streams import StreamsEosTestDriverService, StreamsEosTestJobRunnerService, \
     StreamsComplexEosTestJobRunnerService, StreamsEosTestVerifyRunnerService, StreamsComplexEosTestVerifyRunnerService
 
-
 class StreamsEosTest(KafkaTest):
     """
     Test of Kafka Streams exactly-once semantics
@@ -39,7 +38,6 @@ def __init__(self, test_context):
         self.driver = StreamsEosTestDriverService(test_context, self.kafka)
         self.test_context = test_context
 
-    @ignore
     @cluster(num_nodes=9)
     def test_rebalance_simple(self):
         self.run_rebalance(StreamsEosTestJobRunnerService(self.test_context, self.kafka),
@@ -47,7 +45,6 @@ def test_rebalance_simple(self):
                            StreamsEosTestJobRunnerService(self.test_context, self.kafka),
                            StreamsEosTestVerifyRunnerService(self.test_context, self.kafka))
 
-    @ignore
     @cluster(num_nodes=9)
     def test_rebalance_complex(self):
         self.run_rebalance(StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka),
@@ -63,6 +60,8 @@ def run_rebalance(self, processor1, processor2, processor3, verifier):
 
         self.driver.start()
 
+        processor1.clean_node_enabled = False
+
         self.add_streams(processor1)
         self.add_streams2(processor1, processor2)
         self.add_streams3(processor1, processor2, processor3)
@@ -79,7 +78,6 @@ def run_rebalance(self, processor1, processor2, processor3, verifier):
 
         verifier.node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % verifier.STDOUT_FILE, allow_fail=False)
 
-    @ignore
     @cluster(num_nodes=9)
     def test_failure_and_recovery(self):
         self.run_failure_and_recovery(StreamsEosTestJobRunnerService(self.test_context, self.kafka),
@@ -87,7 +85,6 @@ def test_failure_and_recovery(self):
                                       StreamsEosTestJobRunnerService(self.test_context, self.kafka),
                                       StreamsEosTestVerifyRunnerService(self.test_context, self.kafka))
 
-    @ignore
     @cluster(num_nodes=9)
     def test_failure_and_recovery_complex(self):
         self.run_failure_and_recovery(StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka),
@@ -103,6 +100,8 @@ def run_failure_and_recovery(self, processor1, processor2, processor3, verifier)
 
         self.driver.start()
 
+        processor1.clean_node_enabled = False
+
         self.add_streams(processor1)
         self.add_streams2(processor1, processor2)
         self.add_streams3(processor1, processor2, processor3)
@@ -159,7 +158,6 @@ def abort_streams(self, keep_alive_processor1, keep_alive_processor2, processor_
         self.wait_for_startup(monitor1, keep_alive_processor1)
 
     def wait_for_startup(self, monitor, processor):
-        self.wait_for(monitor, processor, "StateChange: RUNNING -> REBALANCING")
         self.wait_for(monitor, processor, "StateChange: REBALANCING -> RUNNING")
         self.wait_for(monitor, processor, "processed 500 records from topic=data")
 

From a2b6f66fd37a436ebdd5bee1b74ca157c5ddf9c9 Mon Sep 17 00:00:00 2001
From: Yaswanth Kumar 
Date: Sat, 17 Feb 2018 02:54:18 +0530
Subject: [PATCH 0069/1847] KAFKA-6536: Adding versions for
 japicmp-maven-plugin and maven-shade-plugin in quickstart (#4569)

Author: Yaswanth Kumar 

Reviewers: Guozhang Wang , Matthias J. Sax 
---
 streams/quickstart/pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
index b14a9ab65f72d..e7d1873edb5d5 100644
--- a/streams/quickstart/pom.xml
+++ b/streams/quickstart/pom.xml
@@ -64,6 +64,7 @@
             
                 org.apache.maven.plugins
                 maven-shade-plugin
+                3.1.0
                 
                     
                         
@@ -74,6 +75,7 @@
             
                 com.github.siom79.japicmp
                 japicmp-maven-plugin
+                0.11.0
                 
                     true
                 

From 13caded15e21bff450f6f042e86f856046e0f40a Mon Sep 17 00:00:00 2001
From: ying-zheng 
Date: Fri, 16 Feb 2018 15:03:32 -0800
Subject: [PATCH 0070/1847] KAFKA-6430: Add buffer for gzip streams (#4537)

As described in the JIRA ticket, this can double throughput.
---
 .../kafka/common/record/CompressionType.java       | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java b/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
index 16d6e01f097c8..9b3bfc45983a9 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
@@ -20,6 +20,8 @@
 import org.apache.kafka.common.utils.ByteBufferInputStream;
 import org.apache.kafka.common.utils.ByteBufferOutputStream;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.invoke.MethodHandle;
@@ -49,8 +51,10 @@ public InputStream wrapForInput(ByteBuffer buffer, byte messageVersion, BufferSu
         @Override
         public OutputStream wrapForOutput(ByteBufferOutputStream buffer, byte messageVersion) {
             try {
-                // GZIPOutputStream has a default buffer size of 512 bytes, which is too small
-                return new GZIPOutputStream(buffer, 8 * 1024);
+                // Set input buffer (uncompressed) to 16 KB (none by default) and output buffer (compressed) to
+                // 8 KB (0.5 KB by default) to ensure reasonable performance in cases where the caller passes a small
+                // number of bytes to write (potentially a single byte)
+                return new BufferedOutputStream(new GZIPOutputStream(buffer, 8 * 1024), 16 * 1024);
             } catch (Exception e) {
                 throw new KafkaException(e);
             }
@@ -59,7 +63,11 @@ public OutputStream wrapForOutput(ByteBufferOutputStream buffer, byte messageVer
         @Override
         public InputStream wrapForInput(ByteBuffer buffer, byte messageVersion, BufferSupplier decompressionBufferSupplier) {
             try {
-                return new GZIPInputStream(new ByteBufferInputStream(buffer));
+                // Set output buffer (uncompressed) to 16 KB (none by default) and input buffer (compressed) to
+                // 8 KB (0.5 KB by default) to ensure reasonable performance in cases where the caller reads a small
+                // number of bytes (potentially a single byte)
+                return new BufferedInputStream(new GZIPInputStream(new ByteBufferInputStream(buffer), 8 * 1024),
+                        16 * 1024);
             } catch (Exception e) {
                 throw new KafkaException(e);
             }

From b20639db4489a264d25ca5ad6f5b78489cc3fb0f Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Fri, 16 Feb 2018 23:20:16 +0000
Subject: [PATCH 0071/1847] MINOR: Enable deep-iteration to print data in
 DumpLogSegments (#4396)

Enable deep-iteration option when print-data-log is enabled in DumpLogSegments. Otherwise data is not printed.

Reviewers: Jason Gustafson , Ismael Juma 
---
 .../scala/kafka/tools/DumpLogSegments.scala   |  4 +-
 .../kafka/tools/DumpLogSegmentsTest.scala     | 96 +++++++++++++++++++
 docs/upgrade.html                             |  2 +
 3 files changed, 100 insertions(+), 2 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala

diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala
index f6e804d2ed073..2fc203ad37257 100755
--- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala
+++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala
@@ -52,7 +52,7 @@ object DumpLogSegments {
                                   .describedAs("size")
                                   .ofType(classOf[java.lang.Integer])
                                   .defaultsTo(5 * 1024 * 1024)
-    val deepIterationOpt = parser.accepts("deep-iteration", "if set, uses deep instead of shallow iteration.")
+    val deepIterationOpt = parser.accepts("deep-iteration", "if set, uses deep instead of shallow iteration. Automatically set if print-data-log is enabled.")
     val valueDecoderOpt = parser.accepts("value-decoder-class", "if set, used to deserialize the messages. This class should implement kafka.serializer.Decoder trait. Custom jar should be available in kafka/libs directory.")
                                .withOptionalArg()
                                .ofType(classOf[java.lang.String])
@@ -85,7 +85,7 @@ object DumpLogSegments {
 
     val files = options.valueOf(filesOpt).split(",")
     val maxMessageSize = options.valueOf(maxMessageSizeOpt).intValue()
-    val isDeepIteration = options.has(deepIterationOpt)
+    val isDeepIteration = options.has(deepIterationOpt) || printDataLog
 
     val messageParser = if (options.has(offsetsOpt)) {
       new OffsetsMessageParser
diff --git a/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala b/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala
new file mode 100644
index 0000000000000..01f0010440010
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.tools
+
+import java.io.ByteArrayOutputStream
+
+import kafka.log.{ Log, LogConfig, LogManager }
+import kafka.server.{ BrokerTopicStats, LogDirFailureChannel }
+import kafka.utils.{ MockTime, TestUtils }
+import org.apache.kafka.common.record.{ CompressionType, MemoryRecords, SimpleRecord }
+import org.apache.kafka.common.utils.Utils
+import org.junit.Assert._
+import org.junit.{ After, Before, Test }
+
+class DumpLogSegmentsTest {
+
+  val tmpDir = TestUtils.tempDir()
+  val logDir = TestUtils.randomPartitionLogDir(tmpDir)
+  val logFile = s"$logDir/00000000000000000000.log"
+  val time = new MockTime(0, 0)
+
+  @Before
+  def setUp(): Unit = {
+    val log = Log(logDir, LogConfig(), logStartOffset = 0L, recoveryPoint = 0L, scheduler = time.scheduler,
+      time = time, brokerTopicStats = new BrokerTopicStats, maxProducerIdExpirationMs = 60 * 60 * 1000,
+      producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs,
+      logDirFailureChannel = new LogDirFailureChannel(10))
+
+    /* append two messages */
+    log.appendAsLeader(MemoryRecords.withRecords(CompressionType.NONE, 0,
+      new SimpleRecord("hello".getBytes), new SimpleRecord("there".getBytes)), leaderEpoch = 0)
+    log.flush()
+  }
+
+  @After
+  def tearDown(): Unit = {
+    Utils.delete(tmpDir)
+  }
+
+  @Test
+  def testPrintDataLog(): Unit = {
+
+    def verifyRecordsInOutput(args: Array[String]): Unit = {
+      val output = runDumpLogSegments(args)
+      val lines = output.split("\n")
+      assertTrue(s"Data not printed: $output", lines.length > 2)
+      // Verify that the last two lines are message records
+      (0 until 2).foreach { i =>
+        val line = lines(lines.length - 2 + i)
+        assertTrue(s"Not a valid message record: $line", line.startsWith(s"offset: $i position:"))
+      }
+    }
+
+    def verifyNoRecordsInOutput(args: Array[String]): Unit = {
+      val output = runDumpLogSegments(args)
+      assertFalse(s"Data should not have been printed: $output", output.matches("(?s).*offset: [0-9]* position.*"))
+    }
+
+    // Verify that records are printed with --print-data-log even if --deep-iteration is not specified
+    verifyRecordsInOutput(Array("--print-data-log", "--files", logFile))
+    // Verify that records are printed with --print-data-log if --deep-iteration is also specified
+    verifyRecordsInOutput(Array("--print-data-log", "--deep-iteration", "--files", logFile))
+    // Verify that records are printed with --value-decoder even if --print-data-log is not specified
+    verifyRecordsInOutput(Array("--value-decoder-class", "kafka.serializer.StringDecoder", "--files", logFile))
+    // Verify that records are printed with --key-decoder even if --print-data-log is not specified
+    verifyRecordsInOutput(Array("--key-decoder-class", "kafka.serializer.StringDecoder", "--files", logFile))
+    // Verify that records are printed with --deep-iteration even if --print-data-log is not specified
+    verifyRecordsInOutput(Array("--deep-iteration", "--files", logFile))
+
+    // Verify that records are not printed by default
+    verifyNoRecordsInOutput(Array("--files", logFile))
+  }
+
+  private def runDumpLogSegments(args: Array[String]): String = {
+    val outContent = new ByteArrayOutputStream
+    Console.withOut(outContent) {
+      DumpLogSegments.main(args)
+    }
+    outContent.toString
+  }
+}
diff --git a/docs/upgrade.html b/docs/upgrade.html
index b976173c51fd7..b3ae68f3dafa9 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -75,6 +75,8 @@ 
    Notable changes in 1
 	fine-grained timeouts (instead of hard coded retries as in older version).
    • 
     
  • Kafka Streams rebalance time was reduced further making Kafka Streams more responsive.
    • 
     
  • Kafka Connect now supports message headers in both sink and source connectors, and to manipulate them via simple message transforms. Connectors must be changed to explicitly use them. A new HeaderConverter is introduced to control how headers are (de)serialized, and the new "SimpleHeaderConverter" is used by default to use string representations of values.
    • 
+    
  • kafka.tools.DumpLogSegments now automatically sets deep-iteration option if print-data-log is enabled
+        explicitly or implicitly due to any of the other options like decoder.
    • 
 
 
 
    New Protocol Versions
    

From 7c09b9410a63a1dda118b872afcbd2fcdcd56b65 Mon Sep 17 00:00:00 2001
From: "Jiangjie (Becket) Qin" 
Date: Fri, 16 Feb 2018 18:04:21 -0800
Subject: [PATCH 0072/1847] KAFKA-6568; Log cleaner should check partition
 state before removal from inProgress map  (#4580)

The log cleaner should not naively remove the partition from in progress map without checking the partition state. This may cause the other thread calling `LogCleanerManager.abortAndPauseCleaning()` to hang indefinitely.
---
 .../scala/kafka/log/LogCleanerManager.scala   | 35 ++++++++--
 .../kafka/log/LogCleanerManagerTest.scala     | 70 +++++++++++++++++++
 2 files changed, 100 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index c3d3892aef92c..b23107be49145 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -96,6 +96,23 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
     }
   }
 
+  /**
+    * Package private for unit test. Get the cleaning state of the partition.
+    */
+  private[log] def cleaningState(tp: TopicPartition): Option[LogCleaningState] = {
+    inLock(lock) {
+      inProgress.get(tp)
+    }
+  }
+
+  /**
+    * Package private for unit test. Set the cleaning state of the partition.
+    */
+  private[log] def setCleaningState(tp: TopicPartition, state: LogCleaningState): Unit = {
+    inLock(lock) {
+      inProgress.put(tp, state)
+    }
+  }
 
    /**
     * Choose the log to clean next and add it to the in-progress set. We recompute this
@@ -290,11 +307,11 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
    */
   def doneCleaning(topicPartition: TopicPartition, dataDir: File, endOffset: Long) {
     inLock(lock) {
-      inProgress(topicPartition) match {
-        case LogCleaningInProgress =>
+      inProgress.get(topicPartition) match {
+        case Some(LogCleaningInProgress) =>
           updateCheckpoints(dataDir, Option(topicPartition, endOffset))
           inProgress.remove(topicPartition)
-        case LogCleaningAborted =>
+        case Some(LogCleaningAborted) =>
           inProgress.put(topicPartition, LogCleaningPaused)
           pausedCleaningCond.signalAll()
         case s =>
@@ -305,7 +322,15 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
 
   def doneDeleting(topicPartition: TopicPartition): Unit = {
     inLock(lock) {
-      inProgress.remove(topicPartition)
+      inProgress.get(topicPartition) match {
+        case Some(LogCleaningInProgress) =>
+          inProgress.remove(topicPartition)
+        case Some(LogCleaningAborted) =>
+          inProgress.put(topicPartition, LogCleaningPaused)
+          pausedCleaningCond.signalAll()
+        case s =>
+          throw new IllegalStateException(s"In-progress partition $topicPartition cannot be in $s state.")
+      }
     }
   }
 }
@@ -344,7 +369,7 @@ private[log] object LogCleanerManager extends Logging {
         offset
       }
     }
-    
+
     val compactionLagMs = math.max(log.config.compactionLagMs, 0L)
 
     // find first segment that cannot be cleaned
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index 114602919ea38..42a447a2b16b9 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -215,6 +215,76 @@ class LogCleanerManagerTest extends JUnitSuite with Logging {
     assertEquals(4L, cleanableOffsets._2)
   }
 
+  @Test
+  def testDoneCleaning(): Unit = {
+    val logProps = new Properties()
+    logProps.put(LogConfig.SegmentBytesProp, 1024: java.lang.Integer)
+    val log = makeLog(config = LogConfig.fromProps(logConfig.originals, logProps))
+    while(log.numberOfSegments < 8)
+      log.appendAsLeader(records(log.logEndOffset.toInt, log.logEndOffset.toInt, time.milliseconds()), leaderEpoch = 0)
+
+    val cleanerManager: LogCleanerManager = createCleanerManager(log)
+
+    val tp = new TopicPartition("log", 0)
+    try {
+      cleanerManager.doneCleaning(tp, log.dir, 1)
+    } catch {
+      case _ : IllegalStateException =>
+      case _ : Throwable => fail("Should have thrown IllegalStateException.")
+    }
+
+    try {
+      cleanerManager.setCleaningState(tp, LogCleaningPaused)
+      cleanerManager.doneCleaning(tp, log.dir, 1)
+    } catch {
+      case _ : IllegalStateException =>
+      case _ : Throwable => fail("Should have thrown IllegalStateException.")
+    }
+
+    cleanerManager.setCleaningState(tp, LogCleaningInProgress)
+    cleanerManager.doneCleaning(tp, log.dir, 1)
+    assertTrue(cleanerManager.cleaningState(tp).isEmpty)
+    assertTrue(cleanerManager.allCleanerCheckpoints.get(tp).nonEmpty)
+
+    cleanerManager.setCleaningState(tp, LogCleaningAborted)
+    cleanerManager.doneCleaning(tp, log.dir, 1)
+    assertEquals(LogCleaningPaused, cleanerManager.cleaningState(tp).get)
+    assertTrue(cleanerManager.allCleanerCheckpoints.get(tp).nonEmpty)
+  }
+
+  @Test
+  def testDoneDeleting(): Unit = {
+    val records = TestUtils.singletonRecords("test".getBytes, key="test".getBytes)
+    val log: Log = createLog(records.sizeInBytes * 5, LogConfig.Compact + "," + LogConfig.Delete)
+    val cleanerManager: LogCleanerManager = createCleanerManager(log)
+
+    val tp = new TopicPartition("log", 0)
+
+    try {
+      cleanerManager.doneDeleting(tp)
+    } catch {
+      case _ : IllegalStateException =>
+      case _ : Throwable => fail("Should have thrown IllegalStateException.")
+    }
+
+    try {
+      cleanerManager.setCleaningState(tp, LogCleaningPaused)
+      cleanerManager.doneDeleting(tp)
+    } catch {
+      case _ : IllegalStateException =>
+      case _ : Throwable => fail("Should have thrown IllegalStateException.")
+    }
+
+    cleanerManager.setCleaningState(tp, LogCleaningInProgress)
+    cleanerManager.doneDeleting(tp)
+    assertTrue(cleanerManager.cleaningState(tp).isEmpty)
+
+    cleanerManager.setCleaningState(tp, LogCleaningAborted)
+    cleanerManager.doneDeleting(tp)
+    assertEquals(LogCleaningPaused, cleanerManager.cleaningState(tp).get)
+
+  }
+
   private def createCleanerManager(log: Log): LogCleanerManager = {
     val logs = new Pool[TopicPartition, Log]()
     logs.put(new TopicPartition("log", 0), log)

From ee352be9c88663b95b1096b7a294e61857857380 Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Fri, 16 Feb 2018 18:30:12 -0800
Subject: [PATCH 0073/1847] MINOR: Fix bug introduced by adding batch.size
 without default in FileStreamSourceConnector (#4579)

https://github.com/apache/kafka/pull/4356 added `batch.size` config property to `FileStreamSourceConnector` but the property was added as required without a default in config definition (`ConfigDef`). This results in validation error during connector startup.

Unit tests were added for both `FileStreamSourceConnector` and `FileStreamSinkConnector` to avoid such issues in the future.

Reviewers: Randall Hauch , Jason Gustafson 
---
 .../connect/file/FileStreamSinkConnector.java |  4 ++-
 .../file/FileStreamSourceConnector.java       | 30 +++++++---------
 .../connect/file/FileStreamSourceTask.java    | 13 ++-----
 .../file/FileStreamSinkConnectorTest.java     | 11 ++++++
 .../file/FileStreamSourceConnectorTest.java   | 34 +++++++++++++++++--
 .../file/FileStreamSourceTaskTest.java        | 10 +-----
 6 files changed, 63 insertions(+), 39 deletions(-)

diff --git a/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSinkConnector.java b/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSinkConnector.java
index 4ae7f4bb4f424..136e8998c200c 100644
--- a/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSinkConnector.java
+++ b/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSinkConnector.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.file;
 
+import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
 import org.apache.kafka.common.config.ConfigDef.Type;
@@ -47,7 +48,8 @@ public String version() {
 
     @Override
     public void start(Map props) {
-        filename = props.get(FILE_CONFIG);
+        AbstractConfig parsedConfig = new AbstractConfig(CONFIG_DEF, props);
+        filename = parsedConfig.getString(FILE_CONFIG);
     }
 
     @Override
diff --git a/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceConnector.java b/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceConnector.java
index 59006dae4f4e7..74b5f7c09ed23 100644
--- a/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceConnector.java
+++ b/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceConnector.java
@@ -16,12 +16,13 @@
  */
 package org.apache.kafka.connect.file;
 
+import org.apache.kafka.common.config.AbstractConfig;
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
 import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.utils.AppInfoParser;
 import org.apache.kafka.connect.connector.Task;
-import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.source.SourceConnector;
 
 import java.util.ArrayList;
@@ -42,12 +43,13 @@ public class FileStreamSourceConnector extends SourceConnector {
 
     private static final ConfigDef CONFIG_DEF = new ConfigDef()
         .define(FILE_CONFIG, Type.STRING, null, Importance.HIGH, "Source filename. If not specified, the standard input will be used")
-        .define(TOPIC_CONFIG, Type.STRING, Importance.HIGH, "The topic to publish data to")
-        .define(TASK_BATCH_SIZE_CONFIG, Type.INT, Importance.LOW, "The maximum number of records the Source task can read from file one time");
+        .define(TOPIC_CONFIG, Type.LIST, Importance.HIGH, "The topic to publish data to")
+        .define(TASK_BATCH_SIZE_CONFIG, Type.INT, DEFAULT_TASK_BATCH_SIZE, Importance.LOW,
+                "The maximum number of records the Source task can read from file one time");
 
     private String filename;
     private String topic;
-    private int batchSize = DEFAULT_TASK_BATCH_SIZE;
+    private int batchSize;
 
     @Override
     public String version() {
@@ -56,20 +58,14 @@ public String version() {
 
     @Override
     public void start(Map props) {
-        filename = props.get(FILE_CONFIG);
-        topic = props.get(TOPIC_CONFIG);
-        if (topic == null || topic.isEmpty())
-            throw new ConnectException("FileStreamSourceConnector configuration must include 'topic' setting");
-        if (topic.contains(","))
-            throw new ConnectException("FileStreamSourceConnector should only have a single topic when used as a source.");
-
-        if (props.containsKey(TASK_BATCH_SIZE_CONFIG)) {
-            try {
-                batchSize = Integer.parseInt(props.get(TASK_BATCH_SIZE_CONFIG));
-            } catch (NumberFormatException e) {
-                throw new ConnectException("Invalid FileStreamSourceConnector configuration", e);
-            }
+        AbstractConfig parsedConfig = new AbstractConfig(CONFIG_DEF, props);
+        filename = parsedConfig.getString(FILE_CONFIG);
+        List topics = parsedConfig.getList(TOPIC_CONFIG);
+        if (topics.size() != 1) {
+            throw new ConfigException("'topic' in FileStreamSourceConnector configuration requires definition of a single topic");
         }
+        topic = topics.get(0);
+        batchSize = parsedConfig.getInt(TASK_BATCH_SIZE_CONFIG);
     }
 
     @Override
diff --git a/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceTask.java b/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceTask.java
index 482102f785993..17037f2f15a54 100644
--- a/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceTask.java
+++ b/connect/file/src/main/java/org/apache/kafka/connect/file/FileStreamSourceTask.java
@@ -68,17 +68,10 @@ public void start(Map props) {
             streamOffset = null;
             reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
         }
+        // Missing topic or parsing error is not possible because we've parsed the config in the
+        // Connector
         topic = props.get(FileStreamSourceConnector.TOPIC_CONFIG);
-        if (topic == null)
-            throw new ConnectException("FileStreamSourceTask config missing topic setting");
-
-        if (props.containsKey(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG)) {
-            try {
-                batchSize = Integer.parseInt(props.get(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG));
-            } catch (NumberFormatException e) {
-                throw new ConnectException("Invalid FileStreamSourceTask configuration", e);
-            }
-        }
+        batchSize = Integer.parseInt(props.get(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG));
     }
 
     @Override
diff --git a/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSinkConnectorTest.java b/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSinkConnectorTest.java
index c06c99144be91..2348454481648 100644
--- a/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSinkConnectorTest.java
+++ b/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSinkConnectorTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.connect.file;
 
+import org.apache.kafka.common.config.ConfigValue;
 import org.apache.kafka.connect.connector.ConnectorContext;
 import org.apache.kafka.connect.sink.SinkConnector;
 import org.easymock.EasyMockSupport;
@@ -49,6 +50,16 @@ public void setup() {
         sinkProperties.put(FileStreamSinkConnector.FILE_CONFIG, FILENAME);
     }
 
+    @Test
+    public void testConnectorConfigValidation() {
+        replayAll();
+        List configValues = connector.config().validate(sinkProperties);
+        for (ConfigValue val : configValues) {
+            assertEquals("Config property errors: " + val.errorMessages(), 0, val.errorMessages().size());
+        }
+        verifyAll();
+    }
+
     @Test
     public void testSinkTasks() {
         replayAll();
diff --git a/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceConnectorTest.java b/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceConnectorTest.java
index 69a94a894b285..d7efdacd2f62a 100644
--- a/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceConnectorTest.java
+++ b/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceConnectorTest.java
@@ -16,8 +16,9 @@
  */
 package org.apache.kafka.connect.file;
 
+import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.config.ConfigValue;
 import org.apache.kafka.connect.connector.ConnectorContext;
-import org.apache.kafka.connect.errors.ConnectException;
 import org.easymock.EasyMock;
 import org.easymock.EasyMockSupport;
 import org.junit.Before;
@@ -51,6 +52,16 @@ public void setup() {
         sourceProperties.put(FileStreamSourceConnector.FILE_CONFIG, FILENAME);
     }
 
+    @Test
+    public void testConnectorConfigValidation() {
+        replayAll();
+        List configValues = connector.config().validate(sourceProperties);
+        for (ConfigValue val : configValues) {
+            assertEquals("Config property errors: " + val.errorMessages(), 0, val.errorMessages().size());
+        }
+        verifyAll();
+    }
+
     @Test
     public void testSourceTasks() {
         replayAll();
@@ -87,7 +98,7 @@ public void testSourceTasksStdin() {
         EasyMock.verify(ctx);
     }
 
-    @Test(expected = ConnectException.class)
+    @Test(expected = ConfigException.class)
     public void testMultipleSourcesInvalid() {
         sourceProperties.put(FileStreamSourceConnector.TOPIC_CONFIG, MULTIPLE_TOPICS);
         connector.start(sourceProperties);
@@ -102,4 +113,23 @@ public void testTaskClass() {
 
         EasyMock.verify(ctx);
     }
+
+    @Test(expected = ConfigException.class)
+    public void testMissingTopic() {
+        sourceProperties.remove(FileStreamSourceConnector.TOPIC_CONFIG);
+        connector.start(sourceProperties);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testBlankTopic() {
+        // Because of trimming this tests is same as testing for empty string.
+        sourceProperties.put(FileStreamSourceConnector.TOPIC_CONFIG, "     ");
+        connector.start(sourceProperties);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testInvalidBatchSize() {
+        sourceProperties.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, "abcd");
+        connector.start(sourceProperties);
+    }
 }
diff --git a/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceTaskTest.java b/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceTaskTest.java
index 3cb7128e2068a..c1c0a741816d3 100644
--- a/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceTaskTest.java
+++ b/connect/file/src/test/java/org/apache/kafka/connect/file/FileStreamSourceTaskTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.connect.file;
 
-import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.apache.kafka.connect.source.SourceTaskContext;
 import org.apache.kafka.connect.storage.OffsetStorageReader;
@@ -55,6 +54,7 @@ public void setup() throws IOException {
         config = new HashMap<>();
         config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsolutePath());
         config.put(FileStreamSourceConnector.TOPIC_CONFIG, TOPIC);
+        config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE));
         task = new FileStreamSourceTask();
         offsetStorageReader = createMock(OffsetStorageReader.class);
         context = createMock(SourceTaskContext.class);
@@ -151,14 +151,6 @@ public void testBatchSize() throws IOException, InterruptedException {
         task.stop();
     }
 
-    @Test(expected = ConnectException.class)
-    public void testMissingTopic() throws InterruptedException {
-        replay();
-
-        config.remove(FileStreamSourceConnector.TOPIC_CONFIG);
-        task.start(config);
-    }
-
     @Test
     public void testMissingFile() throws InterruptedException {
         replay();

From 1547cf6de86bbb8f7bacc26ab9ab89d4c9ec5566 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 20 Feb 2018 01:32:09 -0800
Subject: [PATCH 0074/1847] KAFKA-6554; Missing lastOffsetDelta validation
 before log append (#4585)

Add validation checks that the offset range is valid and aligned with the batch count prior to appending to the log. Several unit tests have been added to verify the various invalid cases.
---
 .../common/record/DefaultRecordBatch.java     |  4 +-
 .../main/scala/kafka/log/LogValidator.scala   | 18 ++++++--
 .../unit/kafka/log/LogValidatorTest.scala     | 45 +++++++++++++++++++
 3 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
index ff8d3b990ba0f..71e668e45da01 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java
@@ -106,7 +106,7 @@ public class DefaultRecordBatch extends AbstractRecordBatch implements MutableRe
     static final int CRC_LENGTH = 4;
     static final int ATTRIBUTES_OFFSET = CRC_OFFSET + CRC_LENGTH;
     static final int ATTRIBUTE_LENGTH = 2;
-    static final int LAST_OFFSET_DELTA_OFFSET = ATTRIBUTES_OFFSET + ATTRIBUTE_LENGTH;
+    public static final int LAST_OFFSET_DELTA_OFFSET = ATTRIBUTES_OFFSET + ATTRIBUTE_LENGTH;
     static final int LAST_OFFSET_DELTA_LENGTH = 4;
     static final int FIRST_TIMESTAMP_OFFSET = LAST_OFFSET_DELTA_OFFSET + LAST_OFFSET_DELTA_LENGTH;
     static final int FIRST_TIMESTAMP_LENGTH = 8;
@@ -118,7 +118,7 @@ public class DefaultRecordBatch extends AbstractRecordBatch implements MutableRe
     static final int PRODUCER_EPOCH_LENGTH = 2;
     static final int BASE_SEQUENCE_OFFSET = PRODUCER_EPOCH_OFFSET + PRODUCER_EPOCH_LENGTH;
     static final int BASE_SEQUENCE_LENGTH = 4;
-    static final int RECORDS_COUNT_OFFSET = BASE_SEQUENCE_OFFSET + BASE_SEQUENCE_LENGTH;
+    public static final int RECORDS_COUNT_OFFSET = BASE_SEQUENCE_OFFSET + BASE_SEQUENCE_LENGTH;
     static final int RECORDS_COUNT_LENGTH = 4;
     static final int RECORDS_OFFSET = RECORDS_COUNT_OFFSET + RECORDS_COUNT_LENGTH;
     public static final int RECORD_BATCH_OVERHEAD = RECORDS_OFFSET;
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index 15750e9cd065c..1beb2bdda3727 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -74,15 +74,27 @@ private[kafka] object LogValidator extends Logging {
 
   private def validateBatch(batch: RecordBatch, isFromClient: Boolean, toMagic: Byte): Unit = {
     if (isFromClient) {
+      if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) {
+        val countFromOffsets = batch.lastOffset - batch.baseOffset + 1
+        if (countFromOffsets <= 0)
+          throw new InvalidRecordException(s"Batch has an invalid offset range: [${batch.baseOffset}, ${batch.lastOffset}]")
+
+        // v2 and above messages always have a non-null count
+        val count = batch.countOrNull
+        if (count <= 0)
+          throw new InvalidRecordException(s"Invalid reported count for record batch: $count")
+
+        if (countFromOffsets != batch.countOrNull)
+          throw new InvalidRecordException(s"Inconsistent batch offset range [${batch.baseOffset}, ${batch.lastOffset}] " +
+            s"and count of records $count")
+      }
+
       if (batch.hasProducerId && batch.baseSequence < 0)
         throw new InvalidRecordException(s"Invalid sequence number ${batch.baseSequence} in record batch " +
           s"with producerId ${batch.producerId}")
 
       if (batch.isControlBatch)
         throw new InvalidRecordException("Clients are not allowed to write control records")
-
-      if (Option(batch.countOrNull).contains(0))
-        throw new InvalidRecordException("Record batches must contain at least one record")
     }
 
     if (batch.isTransactional && toMagic < RecordBatch.MAGIC_VALUE_V2)
diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
index 131152af4304f..04e89528b122c 100644
--- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala
@@ -26,6 +26,7 @@ import org.apache.kafka.common.utils.Time
 import org.apache.kafka.test.TestUtils
 import org.junit.Assert._
 import org.junit.Test
+import org.scalatest.Assertions.intercept
 
 import scala.collection.JavaConverters._
 
@@ -147,6 +148,50 @@ class LogValidatorTest {
       compressed = true)
   }
 
+  @Test
+  def testInvalidOffsetRangeAndRecordCount(): Unit = {
+    // The batch to be written contains 3 records, so the correct lastOffsetDelta is 2
+    validateRecordBatchWithCountOverrides(lastOffsetDelta = 2, count = 3)
+
+    // Count and offset range are inconsistent or invalid
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 0, count = 3)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 15, count = 3)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = -3, count = 3)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 2, count = -3)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 2, count = 6)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 2, count = 0)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = -3, count = -2)
+
+    // Count and offset range are consistent, but do not match the actual number of records
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 5, count = 6)
+    assertInvalidBatchCountOverrides(lastOffsetDelta = 1, count = 2)
+  }
+
+  private def assertInvalidBatchCountOverrides(lastOffsetDelta: Int, count: Int): Unit = {
+    intercept[InvalidRecordException] {
+      validateRecordBatchWithCountOverrides(lastOffsetDelta, count)
+    }
+  }
+
+  private def validateRecordBatchWithCountOverrides(lastOffsetDelta: Int, count: Int) {
+    val records = createRecords(magicValue = RecordBatch.MAGIC_VALUE_V2, timestamp = 1234L, codec = CompressionType.NONE)
+    records.buffer.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, count)
+    records.buffer.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta)
+    LogValidator.validateMessagesAndAssignOffsets(
+      records,
+      offsetCounter = new LongRef(0),
+      time = time,
+      now = time.milliseconds(),
+      sourceCodec = DefaultCompressionCodec,
+      targetCodec = DefaultCompressionCodec,
+      compactedTopic = false,
+      magic = RecordBatch.MAGIC_VALUE_V2,
+      timestampType = TimestampType.LOG_APPEND_TIME,
+      timestampDiffMaxMs = 1000L,
+      partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH,
+      isFromClient = true)
+  }
+
   @Test
   def testLogAppendTimeWithoutRecompressionV2() {
     checkLogAppendTimeWithoutRecompression(RecordBatch.MAGIC_VALUE_V2)

From f10c0d38634822ab4c9abc4744268c9fd5b50a2c Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 20 Feb 2018 10:45:50 +0000
Subject: [PATCH 0075/1847] MINOR: Fix file source task configs in system
 tests.

Another fall-through of `headers.converter` and `batch.size` properties. Here in `FileStreamSourceConnector` tests

Author: Konstantine Karantasis 

Reviewers: Randall Hauch , Damian Guy 

Closes #4590 from kkonstantine/MINOR-Fix-file-source-task-config-in-system-tests
---
 tests/kafkatest/tests/connect/connect_rest_test.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/kafkatest/tests/connect/connect_rest_test.py b/tests/kafkatest/tests/connect/connect_rest_test.py
index 8172df3b160ce..3c7cd899f069a 100644
--- a/tests/kafkatest/tests/connect/connect_rest_test.py
+++ b/tests/kafkatest/tests/connect/connect_rest_test.py
@@ -31,14 +31,15 @@ class ConnectRestApiTest(KafkaTest):
     FILE_SOURCE_CONNECTOR = 'org.apache.kafka.connect.file.FileStreamSourceConnector'
     FILE_SINK_CONNECTOR = 'org.apache.kafka.connect.file.FileStreamSinkConnector'
 
-    FILE_SOURCE_CONFIGS = {'name', 'connector.class', 'tasks.max', 'key.converter', 'value.converter', 'topic', 'file', 'transforms'}
-    FILE_SINK_CONFIGS = {'name', 'connector.class', 'tasks.max', 'key.converter', 'value.converter', 'topics', 'file', 'transforms', 'topics.regex'}
+    FILE_SOURCE_CONFIGS = {'name', 'connector.class', 'tasks.max', 'key.converter', 'value.converter', 'header.converter', 'batch.size', 'topic', 'file', 'transforms'}
+    FILE_SINK_CONFIGS = {'name', 'connector.class', 'tasks.max', 'key.converter', 'value.converter', 'header.converter', 'topics', 'file', 'transforms', 'topics.regex'}
 
     INPUT_FILE = "/mnt/connect.input"
     INPUT_FILE2 = "/mnt/connect.input2"
     OUTPUT_FILE = "/mnt/connect.output"
 
     TOPIC = "test"
+    DEFAULT_BATCH_SIZE = "2000"
     OFFSETS_TOPIC = "connect-offsets"
     OFFSETS_REPLICATION_FACTOR = "1"
     OFFSETS_PARTITIONS = "1"
@@ -141,7 +142,8 @@ def test_rest_api(self):
             'config': {
                 'task.class': 'org.apache.kafka.connect.file.FileStreamSourceTask',
                 'file': self.INPUT_FILE,
-                'topic': self.TOPIC
+                'topic': self.TOPIC,
+                'batch.size': self.DEFAULT_BATCH_SIZE
             }
         }]
         source_task_info = self.cc.get_connector_tasks("local-file-source")

From a41373309497da449a1f757e2c24399794d77072 Mon Sep 17 00:00:00 2001
From: "Jiangjie (Becket) Qin" 
Date: Tue, 20 Feb 2018 07:19:30 -0800
Subject: [PATCH 0076/1847] KAFKA-6568: Minor clean-ups follow-up (#4592)

Reviewers: Ismael Juma 
---
 .../scala/kafka/log/LogCleanerManager.scala   |  4 +++
 .../kafka/log/LogCleanerManagerTest.scala     | 36 +++++--------------
 2 files changed, 12 insertions(+), 28 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index b23107be49145..223c61196543f 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -314,6 +314,8 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
         case Some(LogCleaningAborted) =>
           inProgress.put(topicPartition, LogCleaningPaused)
           pausedCleaningCond.signalAll()
+        case None =>
+          throw new IllegalStateException(s"State for partition $topicPartition should exist.")
         case s =>
           throw new IllegalStateException(s"In-progress partition $topicPartition cannot be in $s state.")
       }
@@ -328,6 +330,8 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
         case Some(LogCleaningAborted) =>
           inProgress.put(topicPartition, LogCleaningPaused)
           pausedCleaningCond.signalAll()
+        case None =>
+          throw new IllegalStateException(s"State for partition $topicPartition should exist.")
         case s =>
           throw new IllegalStateException(s"In-progress partition $topicPartition cannot be in $s state.")
       }
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
index 42a447a2b16b9..7455763f5b7e3 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala
@@ -226,20 +226,10 @@ class LogCleanerManagerTest extends JUnitSuite with Logging {
     val cleanerManager: LogCleanerManager = createCleanerManager(log)
 
     val tp = new TopicPartition("log", 0)
-    try {
-      cleanerManager.doneCleaning(tp, log.dir, 1)
-    } catch {
-      case _ : IllegalStateException =>
-      case _ : Throwable => fail("Should have thrown IllegalStateException.")
-    }
-
-    try {
-      cleanerManager.setCleaningState(tp, LogCleaningPaused)
-      cleanerManager.doneCleaning(tp, log.dir, 1)
-    } catch {
-      case _ : IllegalStateException =>
-      case _ : Throwable => fail("Should have thrown IllegalStateException.")
-    }
+    intercept[IllegalStateException](cleanerManager.doneCleaning(tp, log.dir, 1))
+
+    cleanerManager.setCleaningState(tp, LogCleaningPaused)
+    intercept[IllegalStateException](cleanerManager.doneCleaning(tp, log.dir, 1))
 
     cleanerManager.setCleaningState(tp, LogCleaningInProgress)
     cleanerManager.doneCleaning(tp, log.dir, 1)
@@ -260,20 +250,10 @@ class LogCleanerManagerTest extends JUnitSuite with Logging {
 
     val tp = new TopicPartition("log", 0)
 
-    try {
-      cleanerManager.doneDeleting(tp)
-    } catch {
-      case _ : IllegalStateException =>
-      case _ : Throwable => fail("Should have thrown IllegalStateException.")
-    }
-
-    try {
-      cleanerManager.setCleaningState(tp, LogCleaningPaused)
-      cleanerManager.doneDeleting(tp)
-    } catch {
-      case _ : IllegalStateException =>
-      case _ : Throwable => fail("Should have thrown IllegalStateException.")
-    }
+    intercept[IllegalStateException](cleanerManager.doneDeleting(tp))
+
+    cleanerManager.setCleaningState(tp, LogCleaningPaused)
+    intercept[IllegalStateException](cleanerManager.doneDeleting(tp))
 
     cleanerManager.setCleaningState(tp, LogCleaningInProgress)
     cleanerManager.doneDeleting(tp)

From ac2536e77e43510cde6009146487a5f867bf9b3f Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Tue, 20 Feb 2018 22:27:21 +0530
Subject: [PATCH 0077/1847] KAFKA-5624; Add expiry check to sensor.add()
 methods (#4404)

---
 .../apache/kafka/common/metrics/Sensor.java   | 25 +++++++++----
 .../producer/internals/BufferPoolTest.java    |  2 +-
 .../kafka/common/metrics/SensorTest.java      | 35 +++++++++++++++++++
 3 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java b/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java
index 837ac2e4b434e..06c8c7f362b60 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/Sensor.java
@@ -207,9 +207,11 @@ public void checkQuotas(long timeMs) {
 
     /**
      * Register a compound statistic with this sensor with no config override
+     * @param stat The stat to register
+     * @return true if stat is added to sensor, false if sensor is expired
      */
-    public void add(CompoundStat stat) {
-        add(stat, null);
+    public boolean add(CompoundStat stat) {
+        return add(stat, null);
     }
 
     /**
@@ -217,8 +219,12 @@ public void add(CompoundStat stat) {
      * @param stat The stat to register
      * @param config The configuration for this stat. If null then the stat will use the default configuration for this
      *        sensor.
+     * @return true if stat is added to sensor, false if sensor is expired
      */
-    public synchronized void add(CompoundStat stat, MetricConfig config) {
+    public synchronized boolean add(CompoundStat stat, MetricConfig config) {
+        if (hasExpired())
+            return false;
+
         this.stats.add(Utils.notNull(stat));
         Object lock = new Object();
         for (NamedMeasurable m : stat.stats()) {
@@ -226,15 +232,17 @@ public synchronized void add(CompoundStat stat, MetricConfig config) {
             this.registry.registerMetric(metric);
             this.metrics.add(metric);
         }
+        return true;
     }
 
     /**
      * Register a metric with this sensor
      * @param metricName The name of the metric
      * @param stat The statistic to keep
+     * @return true if metric is added to sensor, false if sensor is expired
      */
-    public void add(MetricName metricName, MeasurableStat stat) {
-        add(metricName, stat, null);
+    public boolean add(MetricName metricName, MeasurableStat stat) {
+        return add(metricName, stat, null);
     }
 
     /**
@@ -242,8 +250,12 @@ public void add(MetricName metricName, MeasurableStat stat) {
      * @param metricName The name of the metric
      * @param stat The statistic to keep
      * @param config A special configuration for this metric. If null use the sensor default configuration.
+     * @return true if metric is added to sensor, false if sensor is expired
      */
-    public synchronized void add(MetricName metricName, MeasurableStat stat, MetricConfig config) {
+    public synchronized boolean add(MetricName metricName, MeasurableStat stat, MetricConfig config) {
+        if (hasExpired())
+            return false;
+
         KafkaMetric metric = new KafkaMetric(new Object(),
                                              Utils.notNull(metricName),
                                              Utils.notNull(stat),
@@ -252,6 +264,7 @@ public synchronized void add(MetricName metricName, MeasurableStat stat, MetricC
         this.registry.registerMetric(metric);
         this.metrics.add(metric);
         this.stats.add(stat);
+        return true;
     }
 
     /**
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
index 8bcc775df560f..23fc5411b3cc6 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/BufferPoolTest.java
@@ -256,7 +256,7 @@ public void testCleanupMemoryAvailabilityOnMetricsException() throws Exception {
         mockedSensor.record(anyDouble(), anyLong());
         expectLastCall().andThrow(new OutOfMemoryError());
         expect(mockedMetrics.metricName(anyString(), eq(metricGroup), anyString())).andReturn(metricName);
-        mockedSensor.add(new Meter(TimeUnit.NANOSECONDS, rateMetricName, totalMetricName));
+        expect(mockedSensor.add(new Meter(TimeUnit.NANOSECONDS, rateMetricName, totalMetricName))).andReturn(true);
 
         replay(mockedMetrics, mockedSensor, metricName);
 
diff --git a/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java b/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
index d22111e128e25..3f7551ef985e0 100644
--- a/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/metrics/SensorTest.java
@@ -20,7 +20,17 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.Meter;
+import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.SystemTime;
+import org.apache.kafka.common.utils.Time;
 import org.junit.Test;
 
 public class SensorTest {
@@ -59,4 +69,29 @@ public void testShouldRecord() {
             0, Sensor.RecordingLevel.DEBUG);
         assertFalse(debugSensor.shouldRecord());
     }
+
+    @Test
+    public void testExpiredSensor() {
+        MetricConfig config = new MetricConfig();
+        Time mockTime = new MockTime();
+        Metrics metrics =  new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), mockTime, true);
+
+        long inactiveSensorExpirationTimeSeconds = 60L;
+        Sensor sensor = new Sensor(metrics, "sensor", null, config, mockTime,
+            inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO);
+
+        assertTrue(sensor.add(metrics.metricName("test1", "grp1"), new Avg()));
+
+        Map emptyTags = Collections.emptyMap();
+        MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags);
+        MetricName totalMetricName = new MetricName("total", "test", "", emptyTags);
+        Meter meter = new Meter(rateMetricName, totalMetricName);
+        assertTrue(sensor.add(meter));
+
+        mockTime.sleep(TimeUnit.SECONDS.toMillis(inactiveSensorExpirationTimeSeconds + 1));
+        assertFalse(sensor.add(metrics.metricName("test3", "grp1"), new Avg()));
+        assertFalse(sensor.add(meter));
+
+        metrics.close();
+    }
 }

From b79e11bb511e259c8187d865761c3b448391603f Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 20 Feb 2018 17:15:31 +0000
Subject: [PATCH 0078/1847] MINOR: Redirect response code in Connect's
 RestClient to logs instead of stdout

Sending the response code of an http request issued via `RestClient` in Connect to stdout seems like a unconventional choice.

This PR redirects the responds code with a message in the logs at DEBUG level (usually the same level as the one that the caller of `RestClient.httpRequest` uses.

This fix will also fix system tests that broke by outputting this response code to stdout.

Author: Konstantine Karantasis 

Reviewers: Randall Hauch , Damian Guy 

Closes #4591 from kkonstantine/MINOR-Redirect-response-code-in-Connect-RestClient-to-logs-instead-of-stdout
---
 .../org/apache/kafka/connect/runtime/rest/RestClient.java   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
index d500ad27be04a..15e8418a30c90 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java
@@ -82,12 +82,14 @@ public static  HttpResponse httpRequest(String url, String method, Object
             req.method(method);
             req.accept("application/json");
             req.agent("kafka-connect");
-            req.content(new StringContentProvider(serializedBody, StandardCharsets.UTF_8), "application/json");
+            if (serializedBody != null) {
+                req.content(new StringContentProvider(serializedBody, StandardCharsets.UTF_8), "application/json");
+            }
 
             ContentResponse res = req.send();
 
             int responseCode = res.getStatus();
-            System.out.println(responseCode);
+            log.debug("Request's response code: {}", responseCode);
             if (responseCode == HttpStatus.NO_CONTENT_204) {
                 return new HttpResponse<>(responseCode, convertHttpFieldsToMap(res.getHeaders()), null);
             } else if (responseCode >= 400) {

From 57059d40223d6284d1b2c9f3034b30f6bd61c44f Mon Sep 17 00:00:00 2001
From: Damian Guy 
Date: Tue, 20 Feb 2018 17:46:05 +0000
Subject: [PATCH 0079/1847] MINOR: Fix streams broker compatibility test.

Change the string in the test condition to the one that is logged

Author: Damian Guy 

Reviewers: Bill Bejeck , Guozhang Wang 

Closes #4599 from dguy/broker-compatibility
---
 .../tests/streams/streams_broker_compatibility_test.py        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
index 1eb46efea7ed0..b00b9bb25c725 100644
--- a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
+++ b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
@@ -67,9 +67,9 @@ def test_fail_fast_on_incompatible_brokers_if_eos_enabled(self, broker_version):
 
         processor.node.account.ssh(processor.start_cmd(processor.node))
         with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
-            monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ',
+            monitor.wait_until("Exception in thread \"main\" org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ",
                                timeout_sec=60,
-                               err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ' error message " + str(processor.node.account))
+                               err_msg="Exception in thread \"main\" org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS " + str(processor.node.account))
 
         self.kafka.stop()
 

From eaafbdecb56d11f66d833b63fbc2aced65990e69 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Tue, 20 Feb 2018 12:42:05 -0800
Subject: [PATCH 0080/1847] MINOR: Fix logger name override (#4600)

This regressed during the log4j -> scalalogging change.
Added unit tests, one of which failed before the fix.
---
 .../controller/ControllerChannelManager.scala |  1 -
 .../src/main/scala/kafka/log/LogCleaner.scala |  7 +++---
 .../scala/kafka/log/LogCleanerManager.scala   |  2 +-
 .../scala/kafka/server/MetadataCache.scala    |  2 +-
 core/src/main/scala/kafka/utils/Logging.scala | 11 +++++----
 ...gationTokenEndToEndAuthorizationTest.scala |  2 +-
 ...aslScramSslEndToEndAuthorizationTest.scala |  1 -
 ...eListenersWithDefaultJaasContextTest.scala |  2 --
 .../test/scala/kafka/utils/LoggingTest.scala  | 24 +++++++++++++++++++
 9 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 4d7ccf1247f4a..aab4de23606fd 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -36,7 +36,6 @@ import org.apache.kafka.common.security.JaasContext
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.utils.{LogContext, Time}
 import org.apache.kafka.common.{Node, TopicPartition}
-import org.slf4j.event.Level
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index 568d83f292224..07e8440f26f2f 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -20,7 +20,6 @@ package kafka.log
 import java.io.{File, IOException}
 import java.nio._
 import java.nio.file.Files
-import java.util
 import java.util.Date
 import java.util.concurrent.TimeUnit
 
@@ -262,9 +261,9 @@ class LogCleaner(initialConfig: CleanerConfig,
   private class CleanerThread(threadId: Int)
     extends ShutdownableThread(name = "kafka-log-cleaner-thread-" + threadId, isInterruptible = false) {
 
-    override val loggerName = classOf[LogCleaner].getName
+    protected override def loggerName = classOf[LogCleaner].getName
 
-    if(config.dedupeBufferSize / config.numThreads > Int.MaxValue)
+    if (config.dedupeBufferSize / config.numThreads > Int.MaxValue)
       warn("Cannot use more than 2G of cleaner buffer space per cleaner thread, ignoring excess buffer space...")
 
     val cleaner = new Cleaner(id = threadId,
@@ -406,7 +405,7 @@ private[log] class Cleaner(val id: Int,
                            time: Time,
                            checkDone: (TopicPartition) => Unit) extends Logging {
 
-  override val loggerName = classOf[LogCleaner].getName
+  protected override def loggerName = classOf[LogCleaner].getName
 
   this.logIdent = "Cleaner " + id + ": "
 
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 223c61196543f..ba8d7c7e9c07e 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -53,7 +53,7 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
 
   import LogCleanerManager._
 
-  override val loggerName = classOf[LogCleaner].getName
+  protected override def loggerName = classOf[LogCleaner].getName
 
   // package-private for testing
   private[log] val offsetCheckpointFile = "cleaner-offset-checkpoint"
diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala
index eb2d835d876a4..7cdb8f1a1a3c2 100755
--- a/core/src/main/scala/kafka/server/MetadataCache.scala
+++ b/core/src/main/scala/kafka/server/MetadataCache.scala
@@ -23,7 +23,7 @@ import scala.collection.{Seq, Set, mutable}
 import scala.collection.JavaConverters._
 import kafka.cluster.{Broker, EndPoint}
 import kafka.api._
-import kafka.common.{BrokerEndPointNotAvailableException, TopicAndPartition}
+import kafka.common.TopicAndPartition
 import kafka.controller.StateChangeLogger
 import kafka.utils.CoreUtils._
 import kafka.utils.Logging
diff --git a/core/src/main/scala/kafka/utils/Logging.scala b/core/src/main/scala/kafka/utils/Logging.scala
index e409bba3cca7f..0221821e7473a 100755
--- a/core/src/main/scala/kafka/utils/Logging.scala
+++ b/core/src/main/scala/kafka/utils/Logging.scala
@@ -17,8 +17,8 @@
 
 package kafka.utils
 
-import com.typesafe.scalalogging.{LazyLogging, Logger}
-import org.slf4j.{Marker, MarkerFactory}
+import com.typesafe.scalalogging.Logger
+import org.slf4j.{LoggerFactory, Marker, MarkerFactory}
 
 
 object Log4jControllerRegistration {
@@ -38,13 +38,16 @@ private object Logging {
   private val FatalMarker: Marker = MarkerFactory.getMarker("FATAL")
 }
 
-trait Logging extends LazyLogging {
-  def loggerName: String = logger.underlying.getName
+trait Logging {
+
+  protected lazy val logger = Logger(LoggerFactory.getLogger(loggerName))
 
   protected var logIdent: String = _
 
   Log4jControllerRegistration
 
+  protected def loggerName: String = getClass.getName
+
   protected def msgWithLogIdent(msg: String): String =
     if (logIdent == null) msg else logIdent + msg
 
diff --git a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
index efc50491c0e45..24f435e9fb152 100644
--- a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala
@@ -84,7 +84,7 @@ class DelegationTokenEndToEndAuthorizationTest extends EndToEndAuthorizationTest
     config.put(SaslConfigs.SASL_JAAS_CONFIG, clientLoginContext)
 
     val adminClient = AdminClient.create(config.asScala.toMap)
-    var (error, token)  = adminClient.createToken(List())
+    val (error, token)  = adminClient.createToken(List())
 
     //wait for token to reach all the brokers
     TestUtils.waitUntilTrue(() => servers.forall(server => !server.tokenCache.tokens().isEmpty),
diff --git a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
index f07f4b4c49c55..000fc219b4914 100644
--- a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala
@@ -18,7 +18,6 @@ package kafka.api
 
 import org.apache.kafka.common.security.scram.ScramMechanism
 import kafka.utils.JaasTestUtils
-import kafka.utils.ZkUtils
 import kafka.zk.ConfigEntityChangeNotificationZNode
 
 import scala.collection.JavaConverters._
diff --git a/core/src/test/scala/integration/kafka/server/MultipleListenersWithDefaultJaasContextTest.scala b/core/src/test/scala/integration/kafka/server/MultipleListenersWithDefaultJaasContextTest.scala
index 10df84ea4233a..00695ed2bf6ff 100644
--- a/core/src/test/scala/integration/kafka/server/MultipleListenersWithDefaultJaasContextTest.scala
+++ b/core/src/test/scala/integration/kafka/server/MultipleListenersWithDefaultJaasContextTest.scala
@@ -21,8 +21,6 @@ import java.util.Properties
 
 import kafka.api.Both
 import kafka.utils.JaasTestUtils.JaasSection
-import org.apache.kafka.common.network.ListenerName
-
 
 class MultipleListenersWithDefaultJaasContextTest extends MultipleListenersWithSameSecurityProtocolBaseTest {
 
diff --git a/core/src/test/scala/kafka/utils/LoggingTest.scala b/core/src/test/scala/kafka/utils/LoggingTest.scala
index c0600f8d03dc7..b2bdee3ab9de1 100644
--- a/core/src/test/scala/kafka/utils/LoggingTest.scala
+++ b/core/src/test/scala/kafka/utils/LoggingTest.scala
@@ -34,4 +34,28 @@ class LoggingTest extends Logging {
     val instance = mbs.getObjectInstance(log4jControllerName)
     assertEquals("kafka.utils.Log4jController", instance.getClassName)
   }
+
+  @Test
+  def testLogNameOverride(): Unit = {
+    class TestLogging(overriddenLogName: String) extends Logging {
+      // Expose logger
+      def log = logger
+      override def loggerName = overriddenLogName
+    }
+    val overriddenLogName = "OverriddenLogName"
+    val logging = new TestLogging(overriddenLogName)
+
+    assertEquals(overriddenLogName, logging.log.underlying.getName)
+  }
+
+  @Test
+  def testLogName(): Unit = {
+    class TestLogging extends Logging {
+      // Expose logger
+      def log = logger
+    }
+    val logging = new TestLogging
+
+    assertEquals(logging.getClass.getName, logging.log.underlying.getName)
+  }
 }

From 256708dbbb7204e4025f2ca74eceea1170236255 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Tue, 20 Feb 2018 15:46:36 -0500
Subject: [PATCH 0081/1847] KAFKA-4651: improve test coverage of stores (#4555)

Working on increasing the coverage of stores in unit tests.
Started with `InMemoryKeyValueLoggedStore`

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 .../state/internals/RocksDBSessionStore.java  |  28 ----
 .../streams/state/internals/RocksDBStore.java |  22 ---
 .../internals/AbstractKeyValueStoreTest.java  |  33 +++++
 .../InMemoryKeyValueLoggedStoreTest.java      |  24 +---
 .../MeteredKeyValueBytesStoreTest.java        |  14 ++
 .../state/internals/RocksDBStoreTest.java     | 129 ++++++++++--------
 .../streams/state/internals/SegmentsTest.java |   6 +
 7 files changed, 128 insertions(+), 128 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java
index 77b1abbed20c8..c9267dc8fda21 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.state.internals;
 
 import org.apache.kafka.common.serialization.Serde;
-import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.kstream.internals.SessionKeySerde;
@@ -38,33 +37,6 @@ public class RocksDBSessionStore extends WrappedStateStore.AbstractState
     protected StateSerdes serdes;
     protected String topic;
 
-    // this is optimizing the case when this store is already a bytes store, in which we can avoid Bytes.wrap() costs
-    private static class RocksDBSessionBytesStore extends RocksDBSessionStore {
-        RocksDBSessionBytesStore(final SegmentedBytesStore inner) {
-            super(inner, Serdes.Bytes(), Serdes.ByteArray());
-        }
-
-        @Override
-        public KeyValueIterator, byte[]> findSessions(final Bytes key, final long earliestSessionEndTime, final long latestSessionStartTime) {
-            final KeyValueIterator bytesIterator = bytesStore.fetch(key, earliestSessionEndTime, latestSessionStartTime);
-            return WrappedSessionStoreIterator.bytesIterator(bytesIterator, serdes);
-        }
-
-        @Override
-        public void remove(final Windowed key) {
-            bytesStore.remove(SessionKeySerde.bytesToBinary(key));
-        }
-
-        @Override
-        public void put(final Windowed sessionKey, final byte[] aggregate) {
-            bytesStore.put(SessionKeySerde.bytesToBinary(sessionKey), aggregate);
-        }
-    }
-
-    static RocksDBSessionStore bytesStore(final SegmentedBytesStore inner) {
-        return new RocksDBSessionBytesStore(inner);
-    }
-
     RocksDBSessionStore(final SegmentedBytesStore bytesStore,
                         final Serde keySerde,
                         final Serde aggSerde) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index a2e45e00e56df..f54c783aae1ad 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -363,28 +363,6 @@ public synchronized KeyValueIterator all() {
         return rocksDbIterator;
     }
 
-    public synchronized KeyValue first() {
-        validateStoreOpen();
-
-        final RocksIterator innerIter = db.newIterator();
-        innerIter.seekToFirst();
-        final KeyValue pair = new KeyValue<>(new Bytes(innerIter.key()), innerIter.value());
-        innerIter.close();
-
-        return pair;
-    }
-
-    public synchronized KeyValue last() {
-        validateStoreOpen();
-
-        final RocksIterator innerIter = db.newIterator();
-        innerIter.seekToLast();
-        final KeyValue pair = new KeyValue<>(new Bytes(innerIter.key()), innerIter.value());
-        innerIter.close();
-
-        return pair;
-    }
-
     /**
      * Return an approximate count of key-value mappings in this store.
      *
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
index 0583e916511cd..937b1d0b0bfbd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
@@ -30,12 +30,18 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.core.IsEqual.equalTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 
 public abstract class AbstractKeyValueStoreTest {
@@ -341,4 +347,31 @@ public void testSize() {
         store.flush();
         assertEquals(5, store.approximateNumEntries());
     }
+
+    @Test
+    public void shouldPutAll() {
+        List> entries = new ArrayList<>();
+        entries.add(new KeyValue<>(1, "one"));
+        entries.add(new KeyValue<>(2, "two"));
+
+        store.putAll(entries);
+
+        final List> allReturned = new ArrayList<>();
+        final List> expectedReturned = Arrays.asList(KeyValue.pair(1, "one"), KeyValue.pair(2, "two"));
+        final Iterator> iterator = store.all();
+
+        while (iterator.hasNext()) {
+            allReturned.add(iterator.next());
+        }
+        assertThat(allReturned, equalTo(expectedReturned));
+
+    }
+
+    @Test
+    public void shouldDeleteFromStore() {
+        store.put(1, "one");
+        store.put(2, "two");
+        store.delete(2);
+        assertNull(store.get(2));
+    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java
index adaab006cf5f1..0848970cdbe3e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueLoggedStoreTest.java
@@ -17,19 +17,13 @@
 package org.apache.kafka.streams.state.internals;
 
 import org.apache.kafka.common.serialization.Serde;
-import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.StoreBuilder;
 import org.apache.kafka.streams.state.Stores;
-import org.junit.Test;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
 
 public class InMemoryKeyValueLoggedStoreTest extends AbstractKeyValueStoreTest {
 
@@ -37,24 +31,14 @@ public class InMemoryKeyValueLoggedStoreTest extends AbstractKeyValueStoreTest {
     @Override
     protected  KeyValueStore createKeyValueStore(final ProcessorContext context) {
         final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder(
-                Stores.inMemoryKeyValueStore("my-store"),
-                (Serde) context.keySerde(),
-                (Serde) context.valueSerde())
-                .withLoggingEnabled(Collections.singletonMap("retention.ms", "1000"));
+            Stores.inMemoryKeyValueStore("my-store"),
+            (Serde) context.keySerde(),
+            (Serde) context.valueSerde())
+            .withLoggingEnabled(Collections.singletonMap("retention.ms", "1000"));
 
         final StateStore store = storeBuilder.build();
         store.init(context, store);
 
         return (KeyValueStore) store;
     }
-
-    @Test
-    public void shouldPutAll() {
-        List> entries = new ArrayList<>();
-        entries.add(new KeyValue<>(1, "1"));
-        entries.add(new KeyValue<>(2, "2"));
-        store.putAll(entries);
-        assertEquals(store.get(1), "1");
-        assertEquals(store.get(2), "2");
-    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStoreTest.java
index 3b9d13fec02d6..8880007ff2b59 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueBytesStoreTest.java
@@ -193,6 +193,20 @@ public void shouldGetAllFromInnerStoreAndRecordAllMetric() {
         EasyMock.verify(inner);
     }
 
+    @Test
+    public void shouldFlushInnerWhenFlushTimeRecords() {
+        inner.flush();
+        EasyMock.expectLastCall().once();
+        init();
+
+        metered.flush();
+
+        final KafkaMetric metric = metric("flush-rate");
+        assertTrue(metric.value() > 0);
+        EasyMock.verify(inner);
+    }
+
+
     private KafkaMetric metric(final MetricName metricName) {
         return this.metrics.metric(metricName);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
index 49e893b62351f..408733656e25e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
@@ -61,13 +61,13 @@ public class RocksDBStoreTest {
 
     private Serializer stringSerializer = new StringSerializer();
     private Deserializer stringDeserializer = new StringDeserializer();
-    private RocksDBStore subject;
+    private RocksDBStore rocksDBStore;
     private MockProcessorContext context;
     private File dir;
 
     @Before
     public void setUp() {
-        subject = new RocksDBStore("test");
+        rocksDBStore = new RocksDBStore("test");
         dir = TestUtils.tempDirectory();
         context = new MockProcessorContext(dir,
             Serdes.String(),
@@ -78,18 +78,18 @@ public void setUp() {
 
     @After
     public void tearDown() {
-        subject.close();
+        rocksDBStore.close();
     }
 
     @Test
     public void shouldNotThrowExceptionOnRestoreWhenThereIsPreExistingRocksDbFiles() throws Exception {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
 
         final String message = "how can a 4 ounce bird carry a 2lb coconut";
         int intKey = 1;
         for (int i = 0; i < 2000000; i++) {
-            subject.put(new Bytes(stringSerializer.serialize(null, "theKeyIs" + intKey++)),
-                stringSerializer.serialize(null, message));
+            rocksDBStore.put(new Bytes(stringSerializer.serialize(null, "theKeyIs" + intKey++)),
+                             stringSerializer.serialize(null, message));
         }
 
         final List> restoreBytes = new ArrayList<>();
@@ -103,7 +103,7 @@ public void shouldNotThrowExceptionOnRestoreWhenThereIsPreExistingRocksDbFiles()
         assertThat(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "restoredKey")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "restoredKey")))),
             equalTo("restoredValue"));
     }
 
@@ -114,7 +114,7 @@ public void verifyRocksDbConfigSetterIsCalled() {
         configs.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "test-server:9092");
         configs.put(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG, MockRocksDbConfigSetter.class);
         MockRocksDbConfigSetter.called = false;
-        subject.openDB(new MockProcessorContext(tempDir, new StreamsConfig(configs)));
+        rocksDBStore.openDB(new MockProcessorContext(tempDir, new StreamsConfig(configs)));
 
         assertTrue(MockRocksDbConfigSetter.called);
     }
@@ -129,7 +129,7 @@ public void shouldThrowProcessorStateExceptionOnOpeningReadOnlyDir() throws IOEx
             new ThreadCache(new LogContext("testCache "), 0, new MockStreamsMetrics(new Metrics())));
         tmpDir.setReadOnly();
 
-        subject.openDB(tmpContext);
+        rocksDBStore.openDB(tmpContext);
     }
 
     @Test
@@ -145,91 +145,104 @@ public void shouldPutAll() {
             new Bytes(stringSerializer.serialize(null, "3")),
             stringSerializer.serialize(null, "c")));
 
-        subject.init(context, subject);
-        subject.putAll(entries);
-        subject.flush();
+        rocksDBStore.init(context, rocksDBStore);
+        rocksDBStore.putAll(entries);
+        rocksDBStore.flush();
 
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "1")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "1")))),
             "a");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "2")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "2")))),
             "b");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "3")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "3")))),
             "c");
     }
 
     @Test
     public void shouldTogglePrepareForBulkloadSetting() {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         RocksDBStore.RocksDBBatchingRestoreCallback restoreListener =
-            (RocksDBStore.RocksDBBatchingRestoreCallback) subject.batchingStateRestoreCallback;
+            (RocksDBStore.RocksDBBatchingRestoreCallback) rocksDBStore.batchingStateRestoreCallback;
 
         restoreListener.onRestoreStart(null, null, 0, 0);
-        assertTrue("Should have set bulk loading to true", subject.isPrepareForBulkload());
+        assertTrue("Should have set bulk loading to true", rocksDBStore.isPrepareForBulkload());
 
         restoreListener.onRestoreEnd(null, null, 0);
-        assertFalse("Should have set bulk loading to false", subject.isPrepareForBulkload());
+        assertFalse("Should have set bulk loading to false", rocksDBStore.isPrepareForBulkload());
     }
 
     @Test
     public void shouldTogglePrepareForBulkloadSettingWhenPrexistingSstFiles() throws Exception {
         final List> entries = getKeyValueEntries();
 
-        subject.init(context, subject);
-        context.restore(subject.name(), entries);
+        rocksDBStore.init(context, rocksDBStore);
+        context.restore(rocksDBStore.name(), entries);
 
         RocksDBStore.RocksDBBatchingRestoreCallback restoreListener =
-            (RocksDBStore.RocksDBBatchingRestoreCallback) subject.batchingStateRestoreCallback;
+            (RocksDBStore.RocksDBBatchingRestoreCallback) rocksDBStore.batchingStateRestoreCallback;
 
         restoreListener.onRestoreStart(null, null, 0, 0);
-        assertTrue("Should have not set bulk loading to true", subject.isPrepareForBulkload());
+        assertTrue("Should have not set bulk loading to true", rocksDBStore.isPrepareForBulkload());
 
         restoreListener.onRestoreEnd(null, null, 0);
-        assertFalse("Should have set bulk loading to false", subject.isPrepareForBulkload());
+        assertFalse("Should have set bulk loading to false", rocksDBStore.isPrepareForBulkload());
     }
 
     @Test
     public void shouldRestoreAll() throws Exception {
         final List> entries = getKeyValueEntries();
 
-        subject.init(context, subject);
-        context.restore(subject.name(), entries);
+        rocksDBStore.init(context, rocksDBStore);
+        context.restore(rocksDBStore.name(), entries);
 
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "1")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "1")))),
             "a");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "2")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "2")))),
             "b");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "3")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "3")))),
             "c");
     }
 
+    @Test
+    public void shouldPutOnlyIfAbsentValue() throws Exception {
+        rocksDBStore.init(context, rocksDBStore);
+        final Bytes keyBytes = new Bytes(stringSerializer.serialize(null, "one"));
+        final byte[] valueBytes = stringSerializer.serialize(null, "A");
+        final byte[] valueBytesUpdate = stringSerializer.serialize(null, "B");
+
+        rocksDBStore.putIfAbsent(keyBytes, valueBytes);
+        rocksDBStore.putIfAbsent(keyBytes, valueBytesUpdate);
+
+        final String retrievedValue = stringDeserializer.deserialize(null, rocksDBStore.get(keyBytes));
+        assertEquals(retrievedValue, "A");
+    }
 
     @Test
     public void shouldHandleDeletesOnRestoreAll() throws Exception {
         final List> entries = getKeyValueEntries();
         entries.add(new KeyValue<>("1".getBytes("UTF-8"), (byte[]) null));
 
-        subject.init(context, subject);
-        context.restore(subject.name(), entries);
+        rocksDBStore.init(context, rocksDBStore);
+        context.restore(rocksDBStore.name(), entries);
 
-        final KeyValueIterator iterator = subject.all();
+        final KeyValueIterator iterator = rocksDBStore.all();
         final Set keys = new HashSet<>();
 
         while (iterator.hasNext()) {
@@ -250,10 +263,10 @@ public void shouldHandleDeletesAndPutbackOnRestoreAll() throws Exception {
         // this will restore key "1" as WriteBatch applies updates in order
         entries.add(new KeyValue<>("1".getBytes("UTF-8"), "restored".getBytes("UTF-8")));
 
-        subject.init(context, subject);
-        context.restore(subject.name(), entries);
+        rocksDBStore.init(context, rocksDBStore);
+        context.restore(rocksDBStore.name(), entries);
 
-        final KeyValueIterator iterator = subject.all();
+        final KeyValueIterator iterator = rocksDBStore.all();
         final Set keys = new HashSet<>();
 
         while (iterator.hasNext()) {
@@ -265,17 +278,17 @@ public void shouldHandleDeletesAndPutbackOnRestoreAll() throws Exception {
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "1")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "1")))),
             "restored");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "2")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "2")))),
             "b");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "3")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "3")))),
             "c");
     }
 
@@ -283,24 +296,24 @@ public void shouldHandleDeletesAndPutbackOnRestoreAll() throws Exception {
     public void shouldRestoreThenDeleteOnRestoreAll() throws Exception {
         final List> entries = getKeyValueEntries();
 
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
 
-        context.restore(subject.name(), entries);
+        context.restore(rocksDBStore.name(), entries);
 
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "1")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "1")))),
             "a");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "2")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "2")))),
             "b");
         assertEquals(
             stringDeserializer.deserialize(
                 null,
-                subject.get(new Bytes(stringSerializer.serialize(null, "3")))),
+                rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "3")))),
             "c");
 
         entries.clear();
@@ -309,9 +322,9 @@ public void shouldRestoreThenDeleteOnRestoreAll() throws Exception {
         entries.add(new KeyValue<>("3".getBytes("UTF-8"), "c".getBytes("UTF-8")));
         entries.add(new KeyValue<>("1".getBytes("UTF-8"), (byte[]) null));
 
-        context.restore(subject.name(), entries);
+        context.restore(rocksDBStore.name(), entries);
 
-        final KeyValueIterator iterator = subject.all();
+        final KeyValueIterator iterator = rocksDBStore.all();
         final Set keys = new HashSet<>();
 
         while (iterator.hasNext()) {
@@ -325,57 +338,57 @@ public void shouldRestoreThenDeleteOnRestoreAll() throws Exception {
 
     @Test
     public void shouldThrowNullPointerExceptionOnNullPut() {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         try {
-            subject.put(null, stringSerializer.serialize(null, "someVal"));
+            rocksDBStore.put(null, stringSerializer.serialize(null, "someVal"));
             fail("Should have thrown NullPointerException on null put()");
         } catch (NullPointerException e) { }
     }
 
     @Test
     public void shouldThrowNullPointerExceptionOnNullPutAll() {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         try {
-            subject.put(null, stringSerializer.serialize(null, "someVal"));
+            rocksDBStore.put(null, stringSerializer.serialize(null, "someVal"));
             fail("Should have thrown NullPointerException on null put()");
         } catch (NullPointerException e) { }
     }
 
     @Test
     public void shouldThrowNullPointerExceptionOnNullGet() {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         try {
-            subject.get(null);
+            rocksDBStore.get(null);
             fail("Should have thrown NullPointerException on null get()");
         } catch (NullPointerException e) { }
     }
 
     @Test
     public void shouldThrowNullPointerExceptionOnDelete() {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         try {
-            subject.delete(null);
+            rocksDBStore.delete(null);
             fail("Should have thrown NullPointerException on deleting null key");
         } catch (NullPointerException e) { }
     }
 
     @Test
     public void shouldThrowNullPointerExceptionOnRange() {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         try {
-            subject.range(null, new Bytes(stringSerializer.serialize(null, "2")));
+            rocksDBStore.range(null, new Bytes(stringSerializer.serialize(null, "2")));
             fail("Should have thrown NullPointerException on deleting null key");
         } catch (NullPointerException e) { }
     }
 
     @Test(expected = ProcessorStateException.class)
     public void shouldThrowProcessorStateExceptionOnPutDeletedDir() throws IOException {
-        subject.init(context, subject);
+        rocksDBStore.init(context, rocksDBStore);
         Utils.delete(dir);
-        subject.put(
+        rocksDBStore.put(
             new Bytes(stringSerializer.serialize(null, "anyKey")),
             stringSerializer.serialize(null, "anyValue"));
-        subject.flush();
+        rocksDBStore.flush();
     }
 
     public static class MockRocksDbConfigSetter implements RocksDBConfigSetter {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
index 46606decc7cc0..deb26f735c9c3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
@@ -132,6 +132,12 @@ public void shouldGetSegmentForTimestamp() {
         assertEquals(segment, segments.getSegmentForTimestamp(0L));
     }
 
+    @Test
+    public void shouldGetCorrectSegmentString() {
+        final Segment segment = segments.getOrCreateSegment(0, context);
+        assertEquals("Segment(id=0, name=test.0)", segment.toString());
+    }
+
     @Test
     public void shouldCloseAllOpenSegments() {
         final Segment first = segments.getOrCreateSegment(0, context);

From a050eb56f759954498a0a58993d754cebde144d5 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Tue, 20 Feb 2018 13:07:50 -0800
Subject: [PATCH 0082/1847] MINOR: follow up on Streams EOS system tests
 (#4593)

---
 .../apache/kafka/streams/tests/EosTestDriver.java | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java b/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java
index 7c7485d89f5ec..752cdd696eddd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/EosTestDriver.java
@@ -511,7 +511,6 @@ private static void verifyMax(final Map>> inputPerTopicPerPartition,
                                   final Map>> cntPerTopicPerPartition) {
         final StringDeserializer stringDeserializer = new StringDeserializer();
-        final IntegerDeserializer integerDeserializer = new IntegerDeserializer();
         final LongDeserializer longDeserializer = new LongDeserializer();
 
         final HashMap currentSumPerKey = new HashMap<>();
@@ -552,7 +551,7 @@ private static void verifyAllTransactionFinished(final KafkaConsumer partitions = getAllPartitions(consumer, topics);
         consumer.assign(partitions);
         consumer.seekToEnd(partitions);
-        consumer.poll(0);
+        for (final TopicPartition tp : partitions) {
+            System.out.println(tp + " at position " + consumer.position(tp));
+        }
 
         final Properties producerProps = new Properties();
         producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "VerifyProducer");
@@ -591,6 +592,12 @@ public void onCompletion(final RecordMetadata metadata, final Exception exceptio
         long maxWaitTime = System.currentTimeMillis() + MAX_IDLE_TIME_MS;
         while (!partitions.isEmpty() && System.currentTimeMillis() < maxWaitTime) {
             final ConsumerRecords records = consumer.poll(100);
+            if (records.isEmpty()) {
+                System.out.println("No data received.");
+                for (final TopicPartition tp : partitions) {
+                    System.out.println(tp + " at position " + consumer.position(tp));
+                }
+            }
             for (final ConsumerRecord record : records) {
                 maxWaitTime = System.currentTimeMillis() + MAX_IDLE_TIME_MS;
                 final String topic = record.topic();
@@ -604,6 +611,8 @@ public void onCompletion(final RecordMetadata metadata, final Exception exceptio
                         throw new RuntimeException("Post transactions verification failed. Received unexpected verification record: " +
                             "Expected record <'key','value'> from one of " + partitions + " but got"
                             + " <" + key + "," + value + "> [" + record.topic() + ", " + record.partition() + "]");
+                    } else {
+                        System.out.println("Verifying " + tp + " successful.");
                     }
                 } catch (final SerializationException e) {
                     throw new RuntimeException("Post transactions verification failed. Received unexpected verification record: " +

From f3c4240e6cd4237b1160328fb037708de138a05b Mon Sep 17 00:00:00 2001
From: Konstantine Karantasis 
Date: Tue, 20 Feb 2018 13:39:45 -0800
Subject: [PATCH 0083/1847] MINOR: Restore scanning for super types of Connect
 plugins (#4584)

Enabling scans for super types in reflections is required in order to discover Connect plugins.

Reviewers: Randall Hauch , Jason Gustafson 
---
 .../kafka/connect/runtime/isolation/DelegatingClassLoader.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
index b21cdcbfab448..37b795137e100 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java
@@ -273,7 +273,6 @@ private PluginScanResult scanPluginPath(
         builder.setClassLoaders(new ClassLoader[]{loader});
         builder.addUrls(urls);
         builder.setScanners(new SubTypesScanner());
-        builder.setExpandSuperTypes(false);
         builder.useParallelExecutor();
         Reflections reflections = new InternalReflections(builder);
 

From 75df8cc77adc61746b3abd0e48d2c7a343e805f6 Mon Sep 17 00:00:00 2001
From: wushujames 
Date: Tue, 20 Feb 2018 13:53:49 -0800
Subject: [PATCH 0084/1847] MINOR: Add docs for ReplicationBytesInPerSec and
 ReplicationBytesOutPerSec (#4595)

Add docs for the ReplicationBytesInPerSec and ReplicationBytesOutPerSec metrics. These metrics were introduced in KIP-153 (KAFKA-5194) but the docs were not updated.

Built site-docs, and viewed them in a web browser to confirm.

Reviewers: Mickael Maison , Ismael Juma 
---
 docs/ops.html | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/ops.html b/docs/ops.html
index 49c1674c58917..6ffe97653e6ea 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -764,10 +764,15 @@ 
    6.6 Monitoring
    
         
       
       
-        Byte in rate
+        Byte in rate from clients
         kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec
         
       
+      
+        Byte in rate from other brokers
+        kafka.server:type=BrokerTopicMetrics,name=ReplicationBytesInPerSec
+        
+      
       
         Request rate
         kafka.network:type=RequestMetrics,name=RequestsPerSec,request={Produce|FetchConsumer|FetchFollower}
@@ -800,10 +805,15 @@ 
    6.6 Monitoring
    
         Number of records which required message format conversion.
       
       
-        Byte out rate
+        Byte out rate to clients
         kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec
         
       
+      
+        Byte out rate to other brokers
+        kafka.server:type=BrokerTopicMetrics,name=ReplicationBytesOutPerSec
+        
+      
       
         Log flush rate and time
         kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs

From 660c0c0aa33ced5307ee70bfdb78ebde4b978d73 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Wed, 21 Feb 2018 09:38:39 +0000
Subject: [PATCH 0085/1847] KAFKA-6238; Fix inter-broker protocol message
 format compatibility check

This patch fixes a bug in the validation of the inter-broker protocol and the message format version. We should allow the configured message format api version to be greater than the inter-broker protocol api version as long as the actual message format versions are equal. For example, if the message format version is set to 1.0, it is fine for the inter-broker protocol version to be 0.11.0 because they both use message format v2.

I have added a unit test which checks compatibility for all combinations of the message format version and the inter-broker protocol version.

Author: Jason Gustafson 

Reviewers: Ismael Juma 

Closes #4583 from hachikuji/KAFKA-6328-REOPENED
---
 .../kafka/common/record/RecordFormat.java     | 41 +++++++++++++++++
 .../src/main/scala/kafka/api/ApiVersion.scala | 46 ++++++++++++-------
 core/src/main/scala/kafka/log/Log.scala       |  4 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  5 +-
 .../main/scala/kafka/server/KafkaConfig.scala |  9 +++-
 .../scala/kafka/server/ReplicaManager.scala   |  2 +-
 .../scala/unit/kafka/api/ApiVersionTest.scala | 13 ++++++
 .../unit/kafka/server/KafkaConfigTest.scala   | 24 ++++++++++
 docs/upgrade.html                             | 19 ++++----
 9 files changed, 131 insertions(+), 32 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/record/RecordFormat.java

diff --git a/clients/src/main/java/org/apache/kafka/common/record/RecordFormat.java b/clients/src/main/java/org/apache/kafka/common/record/RecordFormat.java
new file mode 100644
index 0000000000000..e71ec599f4190
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/record/RecordFormat.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.record;
+
+public enum RecordFormat {
+    V0(0), V1(1), V2(2);
+
+    public final byte value;
+
+    RecordFormat(int value) {
+        this.value = (byte) value;
+    }
+
+    public static RecordFormat lookup(byte version) {
+        switch (version) {
+            case 0: return V0;
+            case 1: return V1;
+            case 2: return V2;
+            default: throw new IllegalArgumentException("Unknown format version: " + version);
+        }
+    }
+
+    public static RecordFormat current() {
+        return V2;
+    }
+
+}
diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala
index b8329c1ece218..9270a7a3b0120 100644
--- a/core/src/main/scala/kafka/api/ApiVersion.scala
+++ b/core/src/main/scala/kafka/api/ApiVersion.scala
@@ -17,7 +17,7 @@
 
 package kafka.api
 
-import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.record.RecordFormat
 
 /**
  * This class contains the different Kafka versions.
@@ -90,11 +90,23 @@ object ApiVersion {
 
   def latestVersion = versionNameMap.values.max
 
+  def allVersions: Set[ApiVersion] = {
+    versionNameMap.values.toSet
+  }
+
+  def minVersionForMessageFormat(messageFormatVersion: RecordFormat): String = {
+    messageFormatVersion match {
+      case RecordFormat.V0 => "0.8.0"
+      case RecordFormat.V1 => "0.10.0"
+      case RecordFormat.V2 => "0.11.0"
+      case _ => throw new IllegalArgumentException(s"Invalid message format version $messageFormatVersion")
+    }
+  }
 }
 
 sealed trait ApiVersion extends Ordered[ApiVersion] {
   val version: String
-  val messageFormatVersion: Byte
+  val messageFormatVersion: RecordFormat
   val id: Int
 
   override def compare(that: ApiVersion): Int =
@@ -106,90 +118,90 @@ sealed trait ApiVersion extends Ordered[ApiVersion] {
 // Keep the IDs in order of versions
 case object KAFKA_0_8_0 extends ApiVersion {
   val version: String = "0.8.0.X"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V0
+  val messageFormatVersion = RecordFormat.V0
   val id: Int = 0
 }
 
 case object KAFKA_0_8_1 extends ApiVersion {
   val version: String = "0.8.1.X"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V0
+  val messageFormatVersion = RecordFormat.V0
   val id: Int = 1
 }
 
 case object KAFKA_0_8_2 extends ApiVersion {
   val version: String = "0.8.2.X"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V0
+  val messageFormatVersion = RecordFormat.V0
   val id: Int = 2
 }
 
 case object KAFKA_0_9_0 extends ApiVersion {
   val version: String = "0.9.0.X"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V0
+  val messageFormatVersion = RecordFormat.V0
   val id: Int = 3
 }
 
 case object KAFKA_0_10_0_IV0 extends ApiVersion {
   val version: String = "0.10.0-IV0"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V1
+  val messageFormatVersion = RecordFormat.V1
   val id: Int = 4
 }
 
 case object KAFKA_0_10_0_IV1 extends ApiVersion {
   val version: String = "0.10.0-IV1"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V1
+  val messageFormatVersion = RecordFormat.V1
   val id: Int = 5
 }
 
 case object KAFKA_0_10_1_IV0 extends ApiVersion {
   val version: String = "0.10.1-IV0"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V1
+  val messageFormatVersion = RecordFormat.V1
   val id: Int = 6
 }
 
 case object KAFKA_0_10_1_IV1 extends ApiVersion {
   val version: String = "0.10.1-IV1"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V1
+  val messageFormatVersion = RecordFormat.V1
   val id: Int = 7
 }
 
 case object KAFKA_0_10_1_IV2 extends ApiVersion {
   val version: String = "0.10.1-IV2"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V1
+  val messageFormatVersion = RecordFormat.V1
   val id: Int = 8
 }
 
 case object KAFKA_0_10_2_IV0 extends ApiVersion {
   val version: String = "0.10.2-IV0"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V1
+  val messageFormatVersion = RecordFormat.V1
   val id: Int = 9
 }
 
 case object KAFKA_0_11_0_IV0 extends ApiVersion {
   val version: String = "0.11.0-IV0"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V2
+  val messageFormatVersion = RecordFormat.V2
   val id: Int = 10
 }
 
 case object KAFKA_0_11_0_IV1 extends ApiVersion {
   val version: String = "0.11.0-IV1"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V2
+  val messageFormatVersion = RecordFormat.V2
   val id: Int = 11
 }
 
 case object KAFKA_0_11_0_IV2 extends ApiVersion {
   val version: String = "0.11.0-IV2"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V2
+  val messageFormatVersion = RecordFormat.V2
   val id: Int = 12
 }
 
 case object KAFKA_1_0_IV0 extends ApiVersion {
   val version: String = "1.0-IV0"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V2
+  val messageFormatVersion = RecordFormat.V2
   val id: Int = 13
 }
 
 case object KAFKA_1_1_IV0 extends ApiVersion {
   val version: String = "1.1-IV0"
-  val messageFormatVersion: Byte = RecordBatch.MAGIC_VALUE_V2
+  val messageFormatVersion = RecordFormat.V2
   val id: Int = 14
 }
diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala
index fec984cf5e115..257dd8f9ba458 100644
--- a/core/src/main/scala/kafka/log/Log.scala
+++ b/core/src/main/scala/kafka/log/Log.scala
@@ -465,7 +465,7 @@ class Log(@volatile var dir: File,
 
   private def loadProducerState(lastOffset: Long, reloadFromCleanShutdown: Boolean): Unit = lock synchronized {
     checkIfMemoryMappedBufferClosed()
-    val messageFormatVersion = config.messageFormatVersion.messageFormatVersion
+    val messageFormatVersion = config.messageFormatVersion.messageFormatVersion.value
     info(s"Loading producer state from offset $lastOffset for partition $topicPartition with message " +
       s"format version $messageFormatVersion")
 
@@ -663,7 +663,7 @@ class Log(@volatile var dir: File,
               appendInfo.sourceCodec,
               appendInfo.targetCodec,
               config.compact,
-              config.messageFormatVersion.messageFormatVersion,
+              config.messageFormatVersion.messageFormatVersion.value,
               config.messageTimestampType,
               config.messageTimestampDifferenceMaxMs,
               leaderEpoch,
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index b84587f55c293..9e79afa2a5beb 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -59,7 +59,7 @@ import DescribeLogDirsResponse.LogDirInfo
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation}
 
-import scala.collection.{mutable, _}
+import scala.collection._
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.util.{Failure, Success, Try}
@@ -1347,7 +1347,8 @@ class KafkaApis(val requestChannel: RequestChannel,
       if (apiVersionRequest.hasUnsupportedRequestVersion)
         apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception)
       else
-        ApiVersionsResponse.apiVersionsResponse(requestThrottleMs, config.interBrokerProtocolVersion.messageFormatVersion)
+        ApiVersionsResponse.apiVersionsResponse(requestThrottleMs,
+          config.interBrokerProtocolVersion.messageFormatVersion.value)
     }
     sendResponseMaybeThrottle(request, createResponseCallback)
   }
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 529d0e639d9f0..8b2fb1044e0ea 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -1330,8 +1330,13 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
     require(!advertisedListeners.exists(endpoint => endpoint.host=="0.0.0.0"),
       s"${KafkaConfig.AdvertisedListenersProp} cannot use the nonroutable meta-address 0.0.0.0. "+
       s"Use a routable IP address.")
-    require(interBrokerProtocolVersion >= logMessageFormatVersion,
-      s"log.message.format.version $logMessageFormatVersionString cannot be used when inter.broker.protocol.version is set to $interBrokerProtocolVersionString")
+
+    val messageFormatVersion = logMessageFormatVersion.messageFormatVersion
+    require(interBrokerProtocolVersion.messageFormatVersion.value >= messageFormatVersion.value,
+      s"log.message.format.version $logMessageFormatVersionString can only be used when " +
+        "inter.broker.protocol.version is set to version " +
+        s"${ApiVersion.minVersionForMessageFormat(messageFormatVersion)} or higher")
+
     val interBrokerUsesSasl = interBrokerSecurityProtocol == SecurityProtocol.SASL_PLAINTEXT || interBrokerSecurityProtocol == SecurityProtocol.SASL_SSL
     require(!interBrokerUsesSasl || saslInterBrokerHandshakeRequestEnable || saslMechanismInterBrokerProtocol == SaslConfigs.GSSAPI_MECHANISM,
       s"Only GSSAPI mechanism is supported for inter-broker communication with SASL when inter.broker.protocol.version is set to $interBrokerProtocolVersionString")
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index f4bfe39c9ef0f..470842e9d9e9b 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -1002,7 +1002,7 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   def getMagic(topicPartition: TopicPartition): Option[Byte] =
-    getReplica(topicPartition).flatMap(_.log.map(_.config.messageFormatVersion.messageFormatVersion))
+    getReplica(topicPartition).flatMap(_.log.map(_.config.messageFormatVersion.messageFormatVersion.value))
 
   def maybeUpdateMetadataCache(correlationId: Int, updateMetadataRequest: UpdateMetadataRequest) : Seq[TopicPartition] =  {
     replicaStateChangeLock synchronized {
diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
index 6fc69747f5d55..88c9d523f9bc1 100644
--- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
@@ -17,6 +17,7 @@
 
 package kafka.api
 
+import org.apache.kafka.common.record.RecordFormat
 import org.junit.Test
 import org.junit.Assert._
 
@@ -74,4 +75,16 @@ class ApiVersionTest {
     assertEquals(KAFKA_1_0_IV0, ApiVersion("1.0.1"))
   }
 
+  @Test
+  def testMinVersionForMessageFormat(): Unit = {
+    assertEquals("0.8.0", ApiVersion.minVersionForMessageFormat(RecordFormat.V0))
+    assertEquals("0.10.0", ApiVersion.minVersionForMessageFormat(RecordFormat.V1))
+    assertEquals("0.11.0", ApiVersion.minVersionForMessageFormat(RecordFormat.V2))
+
+    // Ensure that all message format versions have a defined min version so that we remember
+    // to update the function
+    for (messageFormatVersion <- RecordFormat.values)
+      assertNotNull(ApiVersion.minVersionForMessageFormat(messageFormatVersion))
+  }
+
 }
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 6b263342118ce..0213c12814e09 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -522,6 +522,30 @@ class KafkaConfigTest {
     }
   }
 
+  @Test
+  def testInterBrokerVersionMessageFormatCompatibility(): Unit = {
+    def buildConfig(interBrokerProtocol: ApiVersion, messageFormat: ApiVersion): KafkaConfig = {
+      val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181)
+      props.put(KafkaConfig.InterBrokerProtocolVersionProp, interBrokerProtocol.version)
+      props.put(KafkaConfig.LogMessageFormatVersionProp, messageFormat.version)
+      KafkaConfig.fromProps(props)
+    }
+
+    ApiVersion.allVersions.foreach { interBrokerVersion =>
+      ApiVersion.allVersions.foreach { messageFormatVersion =>
+        if (interBrokerVersion.messageFormatVersion.value >= messageFormatVersion.messageFormatVersion.value) {
+          val config = buildConfig(interBrokerVersion, messageFormatVersion)
+          assertEquals(messageFormatVersion, config.logMessageFormatVersion)
+          assertEquals(interBrokerVersion, config.interBrokerProtocolVersion)
+        } else {
+          intercept[IllegalArgumentException] {
+            buildConfig(interBrokerVersion, messageFormatVersion)
+          }
+        }
+      }
+    }
+  }
+
   @Test
   def testFromPropsInvalid() {
     def getBaseProperties(): Properties = {
diff --git a/docs/upgrade.html b/docs/upgrade.html
index b3ae68f3dafa9..3ac293d8498dd 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -36,10 +36,10 @@ 
    Upgrading from 0.8.x, 0.9.x, 0.1
             
  • log.message.format.version=CURRENT_MESSAGE_FORMAT_VERSION  (See potential performance impact
                 following the upgrade for the details on what this configuration does.)
    • 
         
-        If you are upgrading from 0.11.0.x and you have not overridden the message format, then you only need to override
+        If you are upgrading from 0.11.0.x or 1.0.x and you have not overridden the message format, then you only need to override
         the inter-broker protocol format.
         
      
-            
    • inter.broker.protocol.version=CURRENT_KAFKA_VERSION (e.g. 0.8.2, 0.9.0, 0.10.0, 0.10.1, 0.10.2, 0.11.0, 1.0).
      • 
+            
    • inter.broker.protocol.version=CURRENT_KAFKA_VERSION (0.11.0 or 1.0).
      • 
         
    
     
     
  •  Upgrade the brokers one at a time: shut down the broker, update the code, and restart it. 
    • 
@@ -106,10 +106,11 @@ 
    Upgrading from 0.8.x, 0.9.x, 0.1
             
  • log.message.format.version=CURRENT_MESSAGE_FORMAT_VERSION  (See potential performance impact
 		following the upgrade for the details on what this configuration does.)
    • 
         
-	If you are upgrading from 0.11.0.x and you have not overridden the message format, then you only need to override
-	the inter-broker protocol format.
+	If you are upgrading from 0.11.0.x and you have not overridden the message format, you must set
+	both the message format version and the inter-broker protocol version to 0.11.0.
         
      
-            
    • inter.broker.protocol.version=CURRENT_KAFKA_VERSION (e.g. 0.8.2, 0.9.0, 0.10.0, 0.10.1, 0.10.2, 0.11.0).
      • 
+            
    • inter.broker.protocol.version=0.11.0
      • 
+            
    • log.message.format.version=0.11.0
      • 
         
    
     
     
  •  Upgrade the brokers one at a time: shut down the broker, update the code, and restart it. 
    • 
@@ -117,9 +118,11 @@ 
    Upgrading from 0.8.x, 0.9.x, 0.1
     
  •  Restart the brokers one by one for the new protocol version to take effect. 
    • 
     
  •  If you have overridden the message format version as instructed above, then you need to do one more rolling restart to
         upgrade it to its latest version. Once all (or most) consumers have been upgraded to 0.11.0 or later,
-        change log.message.format.version to 1.0 on each broker and restart them one by one. Note that the older Scala consumer
-        does not support the new message format introduced in 0.11, so to avoid the performance cost of down-conversion (or to
-        take advantage of exactly once semantics), the newer Java consumer must be used.
    • 
+        change log.message.format.version to 1.0 on each broker and restart them one by one. If you are upgrading from
+        0.11.0 and log.message.format.version is set to 0.11.0, you can update the config and skip the rolling restart.
+        Note that the older Scala consumer does not support the new message format introduced in 0.11, so to avoid the
+        performance cost of down-conversion (or to take advantage of exactly once semantics),
+        the newer Java consumer must be used.
 
 
 
    Additional Upgrade Notes:
    

From 90e0bbec94dd85e1c5b1af0b6426df0a02e5da3f Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Wed, 21 Feb 2018 16:43:25 +0000
Subject: [PATCH 0086/1847] KAFKA-6573: Update brokerInfo in KafkaController on
 listener update (#4603)

Update `KafkaController.brokerInfo` when listeners are updated since this value is used to register the broker in ZooKeeper if there ZK session expires. Also added test to verify values in ZK after session expiry.
---
 .../kafka/controller/KafkaController.scala    |  5 +++++
 .../kafka/server/DynamicBrokerConfig.scala    |  2 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   |  5 ++++-
 .../kafka/zookeeper/ZooKeeperClient.scala     |  2 +-
 .../DynamicBrokerReconfigurationTest.scala    | 21 +++++++++++++++++++
 .../unit/kafka/zk/ZooKeeperTestHarness.scala  | 13 ++++++++++++
 .../kafka/zookeeper/ZooKeeperClientTest.scala | 13 +++---------
 7 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index f36fc798ad6cc..a8707ad887d76 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -190,6 +190,11 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     eventManager.put(controlledShutdownEvent)
   }
 
+  private[kafka] def updateBrokerInfo(newBrokerInfo: BrokerInfo): Unit = {
+    this.brokerInfo = newBrokerInfo
+    zkClient.updateBrokerInfoInZk(newBrokerInfo)
+  }
+
   private def state: ControllerState = eventManager.state
 
   /**
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index ce4b9e75b7c53..3236af01bbb64 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -751,7 +751,7 @@ class DynamicListenerConfig(server: KafkaServer) extends BrokerReconfigurable wi
     if (listenersAdded.nonEmpty)
       server.socketServer.addListeners(listenersAdded)
 
-    server.zkClient.updateBrokerInfoInZk(server.createBrokerInfo)
+    server.kafkaController.updateBrokerInfo(server.createBrokerInfo)
   }
 
   private def listenersToMap(listeners: Seq[EndPoint]): Map[ListenerName, EndPoint] =
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index afc8202b88a38..6545fde30e947 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -35,7 +35,7 @@ import org.apache.kafka.common.security.token.delegation.{DelegationToken, Token
 import org.apache.kafka.common.utils.Time
 import org.apache.zookeeper.KeeperException.{Code, NodeExistsException}
 import org.apache.zookeeper.data.{ACL, Stat}
-import org.apache.zookeeper.{CreateMode, KeeperException}
+import org.apache.zookeeper.{CreateMode, KeeperException, ZooKeeper}
 
 import scala.collection.mutable.ArrayBuffer
 import scala.collection.{Seq, mutable}
@@ -61,6 +61,9 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   import KafkaZkClient._
 
+  // Only for testing
+  private[kafka] def currentZooKeeper: ZooKeeper = zooKeeperClient.currentZooKeeper
+
   /**
    * Create a sequential persistent path. That is, the znode will not be automatically deleted upon client's disconnect
    * and a monotonically increasing number will be appended to its name.
diff --git a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
index 3934fd0ad5d6b..efbd6e898a8ca 100644
--- a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
+++ b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
@@ -317,7 +317,7 @@ class ZooKeeperClient(connectString: String,
   }
 
   // Only for testing
-  private[zookeeper] def currentZooKeeper: ZooKeeper = inReadLock(initializationLock) {
+  private[kafka] def currentZooKeeper: ZooKeeper = inReadLock(initializationLock) {
     zooKeeper
   }
   
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index ee3876231b738..f0bd61a0cccf7 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -599,6 +599,26 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val invalidHost = "192.168.0.1"
     alterAdvertisedListener(adminClient, externalAdminClient, "localhost", invalidHost)
 
+    def validateEndpointsInZooKeeper(server: KafkaServer, endpointMatcher: String => Boolean): Unit = {
+      val brokerInfo = zkClient.getBroker(server.config.brokerId)
+      assertTrue("Broker not registered", brokerInfo.nonEmpty)
+      val endpoints = brokerInfo.get.endPoints.toString
+      assertTrue(s"Endpoint update not saved $endpoints", endpointMatcher(endpoints))
+    }
+
+    // Verify that endpoints have been updated in ZK for all brokers
+    servers.foreach(validateEndpointsInZooKeeper(_, endpoints => endpoints.contains(invalidHost)))
+
+    // Trigger session expiry and ensure that controller registers new advertised listener after expiry
+    val controllerEpoch = zkClient.getControllerEpoch
+    val controllerServer = servers(zkClient.getControllerId.getOrElse(throw new IllegalStateException("No controller")))
+    val controllerZkClient = controllerServer.zkClient
+    val sessionExpiringClient = createZooKeeperClientToTriggerSessionExpiry(controllerZkClient.currentZooKeeper)
+    sessionExpiringClient.close()
+    TestUtils.waitUntilTrue(() => zkClient.getControllerEpoch != controllerEpoch,
+      "Controller not re-elected after ZK session expiry")
+    TestUtils.retry(10000)(validateEndpointsInZooKeeper(controllerServer, endpoints => endpoints.contains(invalidHost)))
+
     // Verify that producer connections fail since advertised listener is invalid
     val bootstrap = bootstrapServers.replaceAll(invalidHost, "localhost") // allow bootstrap connection to succeed
     val producer1 = createProducer(trustStoreFile1, retries = 0, bootstrap = bootstrap)
@@ -606,6 +626,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
     val sendFuture = verifyConnectionFailure(producer1)
 
     alterAdvertisedListener(adminClient, externalAdminClient, invalidHost, "localhost")
+    servers.foreach(validateEndpointsInZooKeeper(_, endpoints => !endpoints.contains(invalidHost)))
 
     // Verify that produce/consume work now
     val producer = createProducer(trustStoreFile1, retries = 0)
diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
index 9e7258315d860..a12229777513d 100755
--- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
+++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala
@@ -33,6 +33,7 @@ import org.apache.kafka.clients.producer.KafkaProducer
 import org.apache.kafka.clients.consumer.internals.AbstractCoordinator
 import kafka.controller.ControllerEventManager
 import org.apache.kafka.common.utils.Time
+import org.apache.zookeeper.{WatchedEvent, Watcher, ZooKeeper}
 
 @Category(Array(classOf[IntegrationTest]))
 abstract class ZooKeeperTestHarness extends JUnitSuite with Logging {
@@ -67,6 +68,18 @@ abstract class ZooKeeperTestHarness extends JUnitSuite with Logging {
       CoreUtils.swallow(zookeeper.shutdown(), this)
     Configuration.setConfiguration(null)
   }
+
+  // Trigger session expiry by reusing the session id in another client
+  def createZooKeeperClientToTriggerSessionExpiry(zooKeeper: ZooKeeper): ZooKeeper = {
+    val dummyWatcher = new Watcher {
+      override def process(event: WatchedEvent): Unit = {}
+    }
+    val anotherZkClient = new ZooKeeper(zkConnect, 1000, dummyWatcher,
+      zooKeeper.getSessionId,
+      zooKeeper.getSessionPasswd)
+    assertNull(anotherZkClient.exists("/nonexistent", false)) // Make sure new client works
+    anotherZkClient
+  }
 }
 
 object ZooKeeperTestHarness {
diff --git a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
index f1c09d7308d73..2e0651c99917e 100644
--- a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala
@@ -29,8 +29,8 @@ import org.apache.kafka.common.utils.Time
 import org.apache.zookeeper.KeeperException.{Code, NoNodeException}
 import org.apache.zookeeper.Watcher.Event.{EventType, KeeperState}
 import org.apache.zookeeper.ZooKeeper.States
-import org.apache.zookeeper.{CreateMode, WatchedEvent, Watcher, ZooDefs, ZooKeeper}
-import org.junit.Assert.{assertArrayEquals, assertEquals, assertFalse, assertNull, assertTrue}
+import org.apache.zookeeper.{CreateMode, WatchedEvent, ZooDefs}
+import org.junit.Assert.{assertArrayEquals, assertEquals, assertFalse, assertTrue}
 import org.junit.{After, Before, Test}
 
 import scala.collection.JavaConverters._
@@ -456,14 +456,7 @@ class ZooKeeperClientTest extends ZooKeeperTestHarness {
       requestThread.start()
       sendCompleteSemaphore.acquire() // Wait for request thread to start processing requests
 
-      // Trigger session expiry by reusing the session id in another client
-      val dummyWatcher = new Watcher {
-        override def process(event: WatchedEvent): Unit = {}
-      }
-      val anotherZkClient = new ZooKeeper(zkConnect, 1000, dummyWatcher,
-        zooKeeperClient.currentZooKeeper.getSessionId,
-        zooKeeperClient.currentZooKeeper.getSessionPasswd)
-      assertNull(anotherZkClient.exists("/nonexistent", false)) // Make sure new client works
+      val anotherZkClient = createZooKeeperClientToTriggerSessionExpiry(zooKeeperClient.currentZooKeeper)
       sendSemaphore.release(maxInflightRequests) // Resume a few more sends which may fail
       anotherZkClient.close()
       sendSemaphore.release(maxInflightRequests) // Resume a few more sends which may fail

From 1667c16be1747ae2ffaab410a5903ad284209456 Mon Sep 17 00:00:00 2001
From: UVN 
Date: Wed, 21 Feb 2018 10:58:35 -0600
Subject: [PATCH 0087/1847] MINOR: Read configuration fields from
 ProducerConfig in example (#4601)

Reading the configuration field names from ProducerConfig class and taking the key and value serializer names from class name directly instead of hardcoding.
---
 examples/src/main/java/kafka/examples/Producer.java | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/examples/src/main/java/kafka/examples/Producer.java b/examples/src/main/java/kafka/examples/Producer.java
index e7be1a078df7f..87212801246d2 100644
--- a/examples/src/main/java/kafka/examples/Producer.java
+++ b/examples/src/main/java/kafka/examples/Producer.java
@@ -20,6 +20,9 @@
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.StringSerializer;
 
 import java.util.Properties;
 import java.util.concurrent.ExecutionException;
@@ -31,10 +34,10 @@ public class Producer extends Thread {
 
     public Producer(String topic, Boolean isAsync) {
         Properties props = new Properties();
-        props.put("bootstrap.servers", KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
-        props.put("client.id", "DemoProducer");
-        props.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
-        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
+        props.put(ProducerConfig.CLIENT_ID_CONFIG, "DemoProducer");
+        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
+        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
         producer = new KafkaProducer<>(props);
         this.topic = topic;
         this.isAsync = isAsync;

From da379c95e4e3fe4dbe4d810a27a2d0b821cb2fb1 Mon Sep 17 00:00:00 2001
From: Ewen Cheslack-Postava 
Date: Wed, 21 Feb 2018 10:52:27 -0800
Subject: [PATCH 0088/1847] MINOR: Cancel port forwarding for
 HttpMetricsCollector during cleanup

Currently port forwarding is setup for HttpMetricsCollector when the Service's start_node method is called, but not canceled during stop. This hasn't presented a problem so far because we don't have tests that use this *and* restart the service. However, if a test/service does that, it will throw an exception since the port is already bound.

This just does the cleanup when stopping so a subsequent attempt to start again will succeed.

https://jenkins.confluent.io/job/system-test-kafka-branch-builder/1320 is a test run for a Test that uses ProducerPerformanceService, which in turn uses HttpMetricsCollector to validate the change.

Author: Ewen Cheslack-Postava 

Reviewers: Ismael Juma , Apurva Mehta 

Closes #4604 from ewencp/cleanup-reverse-port-forward
---
 tests/kafkatest/services/monitor/http.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/kafkatest/services/monitor/http.py b/tests/kafkatest/services/monitor/http.py
index 83324dfd4a12c..88ddb2ef68f3a 100644
--- a/tests/kafkatest/services/monitor/http.py
+++ b/tests/kafkatest/services/monitor/http.py
@@ -173,6 +173,7 @@ def __init__(self, logger, node, remote_port, local_port):
         self.logger = logger
         self._node = node
         self._local_port = local_port
+        self._remote_port = remote_port
 
         self.logger.debug('Forwarding %s port %d to driver port %d', node, remote_port, local_port)
 
@@ -189,6 +190,7 @@ def stop(self):
         self._accept_thread.join(30)
         if self._accept_thread.isAlive():
             raise RuntimeError("Failed to stop reverse forwarder on %s", self._node)
+        self._transport.cancel_port_forward('', self._remote_port)
 
     def _accept(self):
         while not self._stopping:

From 1715e6eac03e26f745b3a6f97a8b61b335207e9d Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Wed, 21 Feb 2018 15:53:46 -0800
Subject: [PATCH 0089/1847] MINOR: Fix Streams-Broker-Compatibility system test
 (#4594)

fixes error message handling for test consumer client and KafkaStreams instance
updates expected error message
fixes race condition in system test code and avoids starting Streams processor twice

Author: Matthias J. Sax 

Reviewer: Bill Bejeck , Guozhang Wang 
---
 .../tests/BrokerCompatibilityTest.java        | 48 +++++++++++--------
 .../streams_broker_compatibility_test.py      | 17 +++----
 2 files changed, 36 insertions(+), 29 deletions(-)

diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java
index b308782a6f48e..af76304a00c69 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java
@@ -31,6 +31,7 @@
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.kstream.Serialized;
 import org.apache.kafka.streams.kstream.ValueMapper;
 import org.apache.kafka.test.TestUtils;
@@ -93,7 +94,13 @@ public String apply(Long value) {
         streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
             @Override
             public void uncaughtException(final Thread t, final Throwable e) {
-                System.err.println("FATAL: An unexpected exception " + e);
+                Throwable cause = e;
+                if (cause instanceof StreamsException) {
+                    while (cause.getCause() != null) {
+                        cause = cause.getCause();
+                    }
+                }
+                System.err.println("FATAL: An unexpected exception " + cause);
                 e.printStackTrace(System.err);
                 System.err.flush();
                 streams.close(30, TimeUnit.SECONDS);
@@ -109,17 +116,20 @@ public void uncaughtException(final Thread t, final Throwable e) {
         producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
         producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
 
-        final KafkaProducer producer = new KafkaProducer<>(producerProperties);
-        producer.send(new ProducerRecord<>(SOURCE_TOPIC, "key", "value"));
-
-
-        System.out.println("wait for result");
-        loopUntilRecordReceived(kafka, eosEnabled);
+        try {
+            try (final KafkaProducer producer = new KafkaProducer<>(producerProperties);) {
+                producer.send(new ProducerRecord<>(SOURCE_TOPIC, "key", "value"));
 
-
-        System.out.println("close Kafka Streams");
-        producer.close();
-        streams.close();
+                System.out.println("wait for result");
+                loopUntilRecordReceived(kafka, eosEnabled);
+                System.out.println("close Kafka Streams");
+                streams.close();
+            }
+        } catch (final RuntimeException e) {
+            System.err.println("Non-Streams exception occurred: ");
+            e.printStackTrace(System.err);
+            System.err.flush();
+        }
     }
 
     private static void loopUntilRecordReceived(final String kafka, final boolean eosEnabled) {
@@ -133,15 +143,15 @@ private static void loopUntilRecordReceived(final String kafka, final boolean eo
             consumerProperties.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT));
         }
 
-        final KafkaConsumer consumer = new KafkaConsumer<>(consumerProperties);
-        consumer.subscribe(Collections.singletonList(SINK_TOPIC));
+        try (final KafkaConsumer consumer = new KafkaConsumer<>(consumerProperties)) {
+            consumer.subscribe(Collections.singletonList(SINK_TOPIC));
 
-        while (true) {
-            final ConsumerRecords records = consumer.poll(100);
-            for (final ConsumerRecord record : records) {
-                if (record.key().equals("key") && record.value().equals("1")) {
-                    consumer.close();
-                    return;
+            while (true) {
+                final ConsumerRecords records = consumer.poll(100);
+                for (final ConsumerRecord record : records) {
+                    if (record.key().equals("key") && record.value().equals("1")) {
+                        return;
+                    }
                 }
             }
         }
diff --git a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
index b00b9bb25c725..5370a3961e316 100644
--- a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
+++ b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
@@ -63,13 +63,12 @@ def test_fail_fast_on_incompatible_brokers_if_eos_enabled(self, broker_version):
         self.kafka.start()
 
         processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, True)
-        processor.start()
 
-        processor.node.account.ssh(processor.start_cmd(processor.node))
         with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
-            monitor.wait_until("Exception in thread \"main\" org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ",
+            processor.start()
+            monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: Cannot create a v0 FindCoordinator request because we require features supported only in 1 or later.',
                                timeout_sec=60,
-                               err_msg="Exception in thread \"main\" org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS " + str(processor.node.account))
+                               err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: Cannot create a v0 FindCoordinator request because we require features supported only in 1 or later.' error message " + str(processor.node.account))
 
         self.kafka.stop()
 
@@ -98,13 +97,12 @@ def test_fail_fast_on_incompatible_brokers(self, broker_version):
         self.kafka.start()
 
         processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False)
-        processor.start()
 
-        processor.node.account.ssh(processor.start_cmd(processor.node))
         with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
-            monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.streams.errors.StreamsException: Could not create topic kafka-streams-system-test-broker-compatibility-KSTREAM-AGGREGATE-STATE-STORE-0000000001-changelog.',
+            processor.start()
+            monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support CREATE_TOPICS',
                         timeout_sec=60,
-                        err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.streams.errors.StreamsException: Could not create topic kafka-streams-system-test-broker-compatibility-KSTREAM-AGGREGATE-STATE-STORE-0000000001-changelog.' error message " + str(processor.node.account))
+                        err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support CREATE_TOPICS' error message " + str(processor.node.account))
 
         self.kafka.stop()
 
@@ -116,10 +114,9 @@ def test_timeout_on_pre_010_brokers(self, broker_version):
         self.kafka.start()
 
         processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False)
-        processor.start()
 
-        processor.node.account.ssh(processor.start_cmd(processor.node))
         with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
+            processor.start()
             monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.BrokerNotFoundException: Could not find any available broker.',
                                timeout_sec=60,
                                err_msg="Never saw 'no available brokers' error message " + str(processor.node.account))

From fc19c3e6f243a8d1b3e27cdc912dc092bbd342e0 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Thu, 22 Feb 2018 09:39:59 +0000
Subject: [PATCH 0090/1847] KAFKA-6577: Fix Connect system tests and add debug
 messages

**NOTE: This should be backported to the `1.1` branch, and is currently a blocker for 1.1.**

The `connect_test.py::ConnectStandaloneFileTest.test_file_source_and_sink` system test is failing with the SASL configuration without a sufficient explanation. During the test, the Connect worker fails to start, but the Connect log contains no useful information. There are actual several things compounding to cause the failure and make it difficult to understand the problem.

First, the `tests/kafkatest/tests/connect/templates/connect_standalone.properties` is only adding in the broker's security configuration with the `producer.` and `consumer.` prefixes, but is not adding them with no prefix. The worker uses the AdminClient to connect to the broker to get the Kafka cluster ID and to manage the three internal topics, and the AdminClient is configured via top-level properties. Because the SASL test requires the clients all connect using SASL, the lack of broker security configs means the AdminClient was attempting and failing to connect to the broker. This is corrected by adding the broker's security configuration to the Connect worker configuration file at the top-level. (This was already being done in the `connect_distributed.properties` file.)

Second, the default `request.timeout.ms` for the AdminClient (and the other clients) is 120 seconds, so the AdminClient was retrying for 120 seconds before it would give up and thrown an error. However, the test was only waiting for 60 seconds before determining that the service failed to start. This can be corrected by setting `request.timeout.ms=10000` in the Connect distributed and standalone worker configurations.

Third, the Connect workers were recently changed to lookup the Kafka cluster ID before it started the herder. This is unlike the older uses of the AdminClient to find and manage the internal topics, where failure to connect was not necessarily logged correctly but nevertheless still skipped over, relying upon broker auto-topic creation to create the internal topics. (This may be why the test did not fail prior to the recent change to always require a successful AdminClient connection.) Although the worker never got this far in its startup process, the fact that we missed such an error since the prior releases means that failure to connect with the AdminClient was not being properly reported.

The `ConnectStandaloneFileTest.test_file_source_and_sink` system tests were run locally prior to this fix, and they failed as with the nightlies. Once these fixes were made, the locally run system tests passed.

Author: Randall Hauch 

Reviewers: Konstantine Karantasis , Ewen Cheslack-Postava 

Closes #4610 from rhauch/kafka-6577-trunk
---
 .../org/apache/kafka/connect/cli/ConnectDistributed.java     | 1 +
 .../java/org/apache/kafka/connect/cli/ConnectStandalone.java | 1 +
 .../kafka/connect/storage/KafkaConfigBackingStore.java       | 1 +
 .../kafka/connect/storage/KafkaOffsetBackingStore.java       | 1 +
 .../kafka/connect/storage/KafkaStatusBackingStore.java       | 1 +
 .../java/org/apache/kafka/connect/util/ConnectUtils.java     | 5 ++++-
 tests/kafkatest/tests/connect/connect_test.py                | 2 +-
 .../tests/connect/templates/connect-distributed.properties   | 3 +++
 .../tests/connect/templates/connect-standalone.properties    | 4 ++++
 9 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java
index 4afa47dda1ae2..3b7ec87f64450 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java
@@ -74,6 +74,7 @@ public static void main(String[] args) throws Exception {
         DistributedConfig config = new DistributedConfig(workerProps);
 
         String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
+        log.debug("Kafka cluster ID: {}", kafkaClusterId);
 
         RestServer rest = new RestServer(config);
         URI advertisedUrl = rest.advertisedUrl();
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java
index 1769905354171..413cb46cf2898 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java
@@ -78,6 +78,7 @@ public static void main(String[] args) throws Exception {
         StandaloneConfig config = new StandaloneConfig(workerProps);
 
         String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
+        log.debug("Kafka cluster ID: {}", kafkaClusterId);
 
         RestServer rest = new RestServer(config);
         URI advertisedUrl = rest.advertisedUrl();
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index b34e48390e168..e51b365cec613 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -432,6 +432,7 @@ private KafkaBasedLog createKafkaBasedLog(String topic, Map createKafkaBasedLog(String topic, Map createKafkaBasedLog(String topic, Map
Date: Thu, 22 Feb 2018 08:19:13 -0800
Subject: [PATCH 0091/1847] MINOR: Refactor GroupMetadataManager
 cleanupGroupMetadata (#4504)

Refactoring avoids the need to call this method with a infinity as current time to remove all group offsets (when manually deleting the group).
---
 .../coordinator/group/GroupCoordinator.scala  | 11 ++++--
 .../coordinator/group/GroupMetadata.scala     |  3 +-
 .../group/GroupMetadataManager.scala          | 39 +++++++++++--------
 3 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 5ae855232ebda..4e605e2d97fed 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -375,9 +375,11 @@ class GroupCoordinator(val brokerId: Int,
       }
 
       if (eligibleGroups.nonEmpty) {
-        groupManager.cleanupGroupMetadata(None, eligibleGroups, Long.MaxValue)
+        val offsetsRemoved = groupManager.cleanupGroupMetadata(eligibleGroups, group => {
+          group.removeAllOffsets()
+        })
         groupErrors ++= eligibleGroups.map(_.groupId -> Errors.NONE).toMap
-        info(s"The following groups were deleted: ${eligibleGroups.map(_.groupId).mkString(", ")}")
+        info(s"The following groups were deleted: ${eligibleGroups.map(_.groupId).mkString(", ")}. A total of $offsetsRemoved offsets were removed.")
       }
 
       groupErrors
@@ -568,7 +570,10 @@ class GroupCoordinator(val brokerId: Int,
   }
 
   def handleDeletedPartitions(topicPartitions: Seq[TopicPartition]) {
-    groupManager.cleanupGroupMetadata(Some(topicPartitions), groupManager.currentGroups, time.milliseconds())
+    val offsetsRemoved = groupManager.cleanupGroupMetadata(groupManager.currentGroups, group => {
+      group.removeOffsets(topicPartitions)
+    })
+    info(s"Removed $offsetsRemoved offsets associated with deleted partitions: ${topicPartitions.mkString(", ")}.")
   }
 
   private def validateGroup(groupId: String): Option[Errors] = {
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index 07d14f40e04e8..2b9c91f61b73c 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -421,9 +421,10 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   def hasPendingOffsetCommitsFromProducer(producerId: Long) =
     pendingTransactionalOffsetCommits.contains(producerId)
 
+  def removeAllOffsets(): immutable.Map[TopicPartition, OffsetAndMetadata] = removeOffsets(offsets.keySet.toSeq)
+
   def removeOffsets(topicPartitions: Seq[TopicPartition]): immutable.Map[TopicPartition, OffsetAndMetadata] = {
     topicPartitions.flatMap { topicPartition =>
-
       pendingOffsetCommits.remove(topicPartition)
       pendingTransactionalOffsetCommits.foreach { case (_, pendingOffsets) =>
         pendingOffsets.remove(topicPartition)
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 3391fc3a221bd..3b79544a50244 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -713,22 +713,27 @@ class GroupMetadataManager(brokerId: Int,
 
   // visible for testing
   private[group] def cleanupGroupMetadata(): Unit = {
-    cleanupGroupMetadata(None, groupMetadataCache.values, time.milliseconds())
+    val startMs = time.milliseconds()
+    val offsetsRemoved = cleanupGroupMetadata(groupMetadataCache.values, group => {
+      group.removeExpiredOffsets(time.milliseconds())
+    })
+    info(s"Removed $offsetsRemoved expired offsets in ${time.milliseconds() - startMs} milliseconds.")
   }
 
-  def cleanupGroupMetadata(deletedTopicPartitions: Option[Seq[TopicPartition]],
-                           groups: Iterable[GroupMetadata],
-                           startMs: Long) {
+  /**
+    * This function is used to clean up group offsets given the groups and also a function that performs the offset deletion.
+    * @param groups Groups whose metadata are to be cleaned up
+    * @param selector A function that implements deletion of (all or part of) group offsets. This function is called while
+    *                 a group lock is held, therefore there is no need for the caller to also obtain a group lock.
+    * @return The cumulative number of offsets removed
+    */
+  def cleanupGroupMetadata(groups: Iterable[GroupMetadata], selector: GroupMetadata => Map[TopicPartition, OffsetAndMetadata]): Int = {
     var offsetsRemoved = 0
 
     groups.foreach { group =>
       val groupId = group.groupId
       val (removedOffsets, groupIsDead, generation) = group.inLock {
-        val removedOffsets = deletedTopicPartitions match {
-          case Some(topicPartitions) => group.removeOffsets(topicPartitions)
-          case None => group.removeExpiredOffsets(startMs)
-        }
-
+        val removedOffsets = selector(group)
         if (group.is(Empty) && !group.hasOffsets) {
           info(s"Group $groupId transitioned to Dead in generation ${group.generationId}")
           group.transitionTo(Dead)
@@ -736,13 +741,13 @@ class GroupMetadataManager(brokerId: Int,
         (removedOffsets, group.is(Dead), group.generationId)
       }
 
-      val offsetsPartition = partitionFor(groupId)
-      val appendPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, offsetsPartition)
-      getMagic(offsetsPartition) match {
-        case Some(magicValue) =>
-          // We always use CREATE_TIME, like the producer. The conversion to LOG_APPEND_TIME (if necessary) happens automatically.
-          val timestampType = TimestampType.CREATE_TIME
-          val timestamp = time.milliseconds()
+    val offsetsPartition = partitionFor(groupId)
+    val appendPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, offsetsPartition)
+    getMagic(offsetsPartition) match {
+      case Some(magicValue) =>
+        // We always use CREATE_TIME, like the producer. The conversion to LOG_APPEND_TIME (if necessary) happens automatically.
+        val timestampType = TimestampType.CREATE_TIME
+        val timestamp = time.milliseconds()
 
           replicaManager.nonOfflinePartition(appendPartition).foreach { partition =>
             val tombstones = ListBuffer.empty[SimpleRecord]
@@ -788,7 +793,7 @@ class GroupMetadataManager(brokerId: Int,
       }
     }
 
-    info(s"Removed $offsetsRemoved expired offsets in ${time.milliseconds() - startMs} milliseconds.")
+    offsetsRemoved
   }
 
   def handleTxnCompletion(producerId: Long, completedPartitions: Set[Int], isCommit: Boolean) {

From 1d8ed875db6ec624cdb8bc15ec677729a42fc13a Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Thu, 22 Feb 2018 09:47:13 -0800
Subject: [PATCH 0092/1847] MINOR: Fix javadoc for consumer offsets lookup APIs
 which do not block indefinitely (#4613)

The blocking time for these APIs is bounded by the request timeout.
---
 .../kafka/clients/consumer/KafkaConsumer.java  | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index ad96ecf0d2a50..3cd034eff769a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1593,19 +1593,17 @@ public Set paused() {
      * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null
      * will be returned for that partition.
      *
-     * Notice that this method may block indefinitely if the partition does not exist.
-     *
      * @param timestampsToSearch the mapping from partition to the timestamp to look up.
      * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater
      *         than or equal to the target timestamp. {@code null} will be returned for the partition if there is no
      *         such message.
      * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
      * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic(s). See the exception for more details
-     * @throws IllegalArgumentException if the target timestamp is negative.
+     * @throws IllegalArgumentException if the target timestamp is negative
      * @throws org.apache.kafka.common.errors.TimeoutException if the offset metadata could not be fetched before
-     *         expiration of the configured request timeout
+     *         expiration of the configured {@code request.timeout.ms}
      * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up
-     *         the offsets by timestamp.
+     *         the offsets by timestamp
      */
     @Override
     public Map offsetsForTimes(Map timestampsToSearch) {
@@ -1627,7 +1625,6 @@ public Map offsetsForTimes(Map
-     * Notice that this method may block indefinitely if the partition does not exist.
      * This method does not change the current consumer position of the partitions.
      *
      * @see #seekToBeginning(Collection)
@@ -1636,8 +1633,8 @@ public Map offsetsForTimes(Map beginningOffsets(Collection partitions) {
@@ -1655,7 +1652,6 @@ public Map beginningOffsets(Collection par
      * to the the partition, the offset returned will be 0.
      *
      * 
    
-     * Notice that this method may block indefinitely if the partition does not exist.
      * This method does not change the current consumer position of the partitions.
      * 
    
      * When {@code isolation.level=read_committed} the last offset will be the Last Stable Offset (LSO).
@@ -1668,8 +1664,8 @@ public Map beginningOffsets(Collection par
      * @return The end offsets for the given partitions.
      * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
      * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic(s). See the exception for more details
-     * @throws org.apache.kafka.common.errors.TimeoutException if the offset metadata could not be fetched before
-     *         expiration of the configured request timeout
+     * @throws org.apache.kafka.common.errors.TimeoutException if the offsets could not be fetched before
+     *         expiration of the configured {@code request.timeout.ms}
      */
     @Override
     public Map endOffsets(Collection partitions) {

From 66039b1312be6356a3cb12efa8e562d2264498d1 Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Thu, 22 Feb 2018 22:23:14 -0800
Subject: [PATCH 0093/1847] MINOR: Fix ConcurrentModificationException in
 TransactionManager (#4608)

---
 checkstyle/suppressions.xml                   |  2 +-
 .../internals/TransactionManager.java         |  6 +++--
 .../internals/TransactionManagerTest.java     | 27 +++++++++++++++++++
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index de1bdfd4b04ea..f23805e665f93 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -54,7 +54,7 @@
               files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler).java"/>
 
     
+              files="AbstractRequest.java|KerberosLogin.java|WorkerSinkTaskTest.java|TransactionManagerTest.java"/>
 
     
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
index 006a12b1bfd4c..b242d5a65a639 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@ -51,6 +51,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
@@ -568,14 +569,15 @@ synchronized boolean shouldResetProducerStateAfterResolvingSequences() {
         if (isTransactional())
             // We should not reset producer state if we are transactional. We will transition to a fatal error instead.
             return false;
-        for (TopicPartition topicPartition : partitionsWithUnresolvedSequences) {
+        for (Iterator iter = partitionsWithUnresolvedSequences.iterator(); iter.hasNext(); ) {
+            TopicPartition topicPartition = iter.next();
             if (!hasInflightBatches(topicPartition)) {
                 // The partition has been fully drained. At this point, the last ack'd sequence should be once less than
                 // next sequence destined for the partition. If so, the partition is fully resolved. If not, we should
                 // reset the sequence number if necessary.
                 if (isNextSequence(topicPartition, sequenceNumber(topicPartition))) {
                     // This would happen when a batch was expired, but subsequent batches succeeded.
-                    partitionsWithUnresolvedSequences.remove(topicPartition);
+                    iter.remove();
                 } else {
                     // We would enter this branch if all in flight batches were ultimately expired in the producer.
                     log.info("No inflight batches remaining for {}, last ack'd sequence for partition is {}, next sequence is {}. " +
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
index fab139ab816c5..6fcf48059672a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -2213,6 +2213,33 @@ public void testTransitionToFatalErrorWhenRetriedBatchIsExpired() throws Interru
         assertFalse(transactionManager.hasOngoingTransaction());
     }
 
+    @Test
+    public void testShouldResetProducerStateAfterResolvingSequences() throws InterruptedException, ExecutionException {
+        // Create a TransactionManager without a transactionalId to test
+        // shouldResetProducerStateAfterResolvingSequences.
+        TransactionManager manager = new TransactionManager(logContext, null, transactionTimeoutMs,
+            DEFAULT_RETRY_BACKOFF_MS);
+        assertFalse(manager.shouldResetProducerStateAfterResolvingSequences());
+        TopicPartition tp0 = new TopicPartition("foo", 0);
+        TopicPartition tp1 = new TopicPartition("foo", 1);
+        assertEquals(Integer.valueOf(0), manager.sequenceNumber(tp0));
+        assertEquals(Integer.valueOf(0), manager.sequenceNumber(tp1));
+
+        manager.incrementSequenceNumber(tp0, 1);
+        manager.incrementSequenceNumber(tp1, 1);
+        manager.maybeUpdateLastAckedSequence(tp0, 0);
+        manager.maybeUpdateLastAckedSequence(tp1, 0);
+        manager.markSequenceUnresolved(tp0);
+        manager.markSequenceUnresolved(tp1);
+        assertFalse(manager.shouldResetProducerStateAfterResolvingSequences());
+
+        manager.maybeUpdateLastAckedSequence(tp0, 5);
+        manager.incrementSequenceNumber(tp0, 1);
+        manager.markSequenceUnresolved(tp0);
+        manager.markSequenceUnresolved(tp1);
+        assertTrue(manager.shouldResetProducerStateAfterResolvingSequences());
+    }
+
     private void verifyAddPartitionsFailsWithPartitionLevelError(final Errors error) throws InterruptedException {
         final long pid = 1L;
         final short epoch = 1;

From 0e22fd6f8d49e7884b7126d963af12ab305d5629 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Fri, 23 Feb 2018 00:29:49 -0600
Subject: [PATCH 0094/1847] KAFKA-6578: Changed the Connect distributed and
 standalone main method to log all exceptions (#4609)

Any exception thrown by calls within a `main()` method are not logged unless explicitly done so. This change simply adds a try-catch block around most of the content of the distributed and standalone `main()` methods.
---
 .../kafka/connect/cli/ConnectDistributed.java | 81 ++++++++--------
 .../kafka/connect/cli/ConnectStandalone.java  | 93 ++++++++++---------
 2 files changed, 94 insertions(+), 80 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java
index 3b7ec87f64450..54854fe4b8077 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java
@@ -58,52 +58,59 @@ public static void main(String[] args) throws Exception {
             Exit.exit(1);
         }
 
-        Time time = Time.SYSTEM;
-        log.info("Kafka Connect distributed worker initializing ...");
-        long initStart = time.hiResClockMs();
-        WorkerInfo initInfo = new WorkerInfo();
-        initInfo.logAll();
+        try {
+            Time time = Time.SYSTEM;
+            log.info("Kafka Connect distributed worker initializing ...");
+            long initStart = time.hiResClockMs();
+            WorkerInfo initInfo = new WorkerInfo();
+            initInfo.logAll();
 
-        String workerPropsFile = args[0];
-        Map workerProps = !workerPropsFile.isEmpty() ?
-                Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.emptyMap();
+            String workerPropsFile = args[0];
+            Map workerProps = !workerPropsFile.isEmpty() ?
+                    Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.emptyMap();
 
-        log.info("Scanning for plugin classes. This might take a moment ...");
-        Plugins plugins = new Plugins(workerProps);
-        plugins.compareAndSwapWithDelegatingLoader();
-        DistributedConfig config = new DistributedConfig(workerProps);
+            log.info("Scanning for plugin classes. This might take a moment ...");
+            Plugins plugins = new Plugins(workerProps);
+            plugins.compareAndSwapWithDelegatingLoader();
+            DistributedConfig config = new DistributedConfig(workerProps);
 
-        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
-        log.debug("Kafka cluster ID: {}", kafkaClusterId);
+            String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
+            log.debug("Kafka cluster ID: {}", kafkaClusterId);
 
-        RestServer rest = new RestServer(config);
-        URI advertisedUrl = rest.advertisedUrl();
-        String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
+            RestServer rest = new RestServer(config);
+            URI advertisedUrl = rest.advertisedUrl();
+            String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
 
-        KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
-        offsetBackingStore.configure(config);
+            KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
+            offsetBackingStore.configure(config);
 
-        Worker worker = new Worker(workerId, time, plugins, config, offsetBackingStore);
+            Worker worker = new Worker(workerId, time, plugins, config, offsetBackingStore);
 
-        Converter internalValueConverter = worker.getInternalValueConverter();
-        StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
-        statusBackingStore.configure(config);
+            Converter internalValueConverter = worker.getInternalValueConverter();
+            StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
+            statusBackingStore.configure(config);
 
-        ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(internalValueConverter, config);
+            ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(internalValueConverter, config);
 
-        DistributedHerder herder = new DistributedHerder(config, time, worker,
-                kafkaClusterId, statusBackingStore, configBackingStore,
-                advertisedUrl.toString());
-        final Connect connect = new Connect(herder, rest);
-        log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart);
-        try {
-            connect.start();
-        } catch (Exception e) {
-            log.error("Failed to start Connect", e);
-            connect.stop();
-        }
+            DistributedHerder herder = new DistributedHerder(config, time, worker,
+                    kafkaClusterId, statusBackingStore, configBackingStore,
+                    advertisedUrl.toString());
+            final Connect connect = new Connect(herder, rest);
+            log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart);
+            try {
+                connect.start();
+            } catch (Exception e) {
+                log.error("Failed to start Connect", e);
+                connect.stop();
+                Exit.exit(3);
+            }
 
-        // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
-        connect.awaitStop();
+            // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
+            connect.awaitStop();
+
+        } catch (Throwable t) {
+            log.error("Stopping due to error", t);
+            Exit.exit(2);
+        }
     }
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java
index 413cb46cf2898..aba9d9c32aa4b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java
@@ -62,58 +62,65 @@ public static void main(String[] args) throws Exception {
             Exit.exit(1);
         }
 
-        Time time = Time.SYSTEM;
-        log.info("Kafka Connect standalone worker initializing ...");
-        long initStart = time.hiResClockMs();
-        WorkerInfo initInfo = new WorkerInfo();
-        initInfo.logAll();
+        try {
+            Time time = Time.SYSTEM;
+            log.info("Kafka Connect standalone worker initializing ...");
+            long initStart = time.hiResClockMs();
+            WorkerInfo initInfo = new WorkerInfo();
+            initInfo.logAll();
 
-        String workerPropsFile = args[0];
-        Map workerProps = !workerPropsFile.isEmpty() ?
-                Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.emptyMap();
+            String workerPropsFile = args[0];
+            Map workerProps = !workerPropsFile.isEmpty() ?
+                    Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.emptyMap();
 
-        log.info("Scanning for plugin classes. This might take a moment ...");
-        Plugins plugins = new Plugins(workerProps);
-        plugins.compareAndSwapWithDelegatingLoader();
-        StandaloneConfig config = new StandaloneConfig(workerProps);
+            log.info("Scanning for plugin classes. This might take a moment ...");
+            Plugins plugins = new Plugins(workerProps);
+            plugins.compareAndSwapWithDelegatingLoader();
+            StandaloneConfig config = new StandaloneConfig(workerProps);
 
-        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
-        log.debug("Kafka cluster ID: {}", kafkaClusterId);
+            String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
+            log.debug("Kafka cluster ID: {}", kafkaClusterId);
 
-        RestServer rest = new RestServer(config);
-        URI advertisedUrl = rest.advertisedUrl();
-        String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
+            RestServer rest = new RestServer(config);
+            URI advertisedUrl = rest.advertisedUrl();
+            String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
 
-        Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore());
+            Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore());
 
-        Herder herder = new StandaloneHerder(worker, kafkaClusterId);
-        final Connect connect = new Connect(herder, rest);
-        log.info("Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart);
+            Herder herder = new StandaloneHerder(worker, kafkaClusterId);
+            final Connect connect = new Connect(herder, rest);
+            log.info("Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart);
 
-        try {
-            connect.start();
-            for (final String connectorPropsFile : Arrays.copyOfRange(args, 1, args.length)) {
-                Map connectorProps = Utils.propsToStringMap(Utils.loadProps(connectorPropsFile));
-                FutureCallback> cb = new FutureCallback<>(new Callback>() {
-                    @Override
-                    public void onCompletion(Throwable error, Herder.Created info) {
-                        if (error != null)
-                            log.error("Failed to create job for {}", connectorPropsFile);
-                        else
-                            log.info("Created connector {}", info.result().name());
-                    }
-                });
-                herder.putConnectorConfig(
-                        connectorProps.get(ConnectorConfig.NAME_CONFIG),
-                        connectorProps, false, cb);
-                cb.get();
+            try {
+                connect.start();
+                for (final String connectorPropsFile : Arrays.copyOfRange(args, 1, args.length)) {
+                    Map connectorProps = Utils.propsToStringMap(Utils.loadProps(connectorPropsFile));
+                    FutureCallback> cb = new FutureCallback<>(new Callback>() {
+                        @Override
+                        public void onCompletion(Throwable error, Herder.Created info) {
+                            if (error != null)
+                                log.error("Failed to create job for {}", connectorPropsFile);
+                            else
+                                log.info("Created connector {}", info.result().name());
+                        }
+                    });
+                    herder.putConnectorConfig(
+                            connectorProps.get(ConnectorConfig.NAME_CONFIG),
+                            connectorProps, false, cb);
+                    cb.get();
+                }
+            } catch (Throwable t) {
+                log.error("Stopping after connector error", t);
+                connect.stop();
+                Exit.exit(3);
             }
+
+            // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
+            connect.awaitStop();
+
         } catch (Throwable t) {
-            log.error("Stopping after connector error", t);
-            connect.stop();
+            log.error("Stopping due to error", t);
+            Exit.exit(2);
         }
-
-        // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
-        connect.awaitStop();
     }
 }

From e26d0d760466c3da6189637a7de69eb5509e7b51 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 23 Feb 2018 15:31:50 -0800
Subject: [PATCH 0095/1847] MINOR: Revert incompatible behavior change to
 consumer reset tool (#4611)

This patch reverts the removal of the --execute option in the offset reset tool and the change to the default behavior when no options were present. For consistency, this patch adds the --execute flag to the streams reset tool, but keeps its current default behavior. A note has been added to both of these commands to warn the user that future default behavior will be to prompt before acting.

Test cases were not actually validating that offsets were committed when the --execute option was present, so I have fixed that and added basic assertions for the dry-run behavior. I also removed some duplicated test boilerplate.

Reviewers: Matthias J. Sax , Guozhang Wang 
---
 .../kafka/admin/ConsumerGroupCommand.scala    | 112 ++--
 .../scala/kafka/tools/StreamsResetter.java    |  56 +-
 .../admin/ConsumerGroupCommandTest.scala      |  19 +
 .../admin/ResetConsumerGroupOffsetTest.scala  | 530 ++++++------------
 .../AbstractResetIntegrationTest.java         |  17 +-
 .../streams/tools/StreamsResetterTest.java    |   3 -
 6 files changed, 309 insertions(+), 428 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
index 4c2d6c7878f5d..68186315a4220 100755
--- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
@@ -114,16 +114,14 @@ object ConsumerGroupCommand extends Logging {
   }
 
   def printOffsetsToReset(groupAssignmentsToReset: Map[TopicPartition, OffsetAndMetadata]): Unit = {
-    print("\n%-30s %-10s %-15s".format("TOPIC", "PARTITION", "NEW-OFFSET"))
-    println()
+    println("\n%-30s %-10s %-15s".format("TOPIC", "PARTITION", "NEW-OFFSET"))
 
     groupAssignmentsToReset.foreach {
       case (consumerAssignment, offsetAndMetadata) =>
-        print("%-30s %-10s %-15s".format(
-          consumerAssignment.topic(),
-          consumerAssignment.partition(),
-          offsetAndMetadata.offset()))
-        println()
+        println("%-30s %-10s %-15s".format(
+          consumerAssignment.topic,
+          consumerAssignment.partition,
+          offsetAndMetadata.offset))
     }
   }
 
@@ -284,7 +282,7 @@ object ConsumerGroupCommand extends Logging {
     protected def opts: ConsumerGroupCommandOptions
 
     protected def getLogEndOffset(topicPartition: TopicPartition): LogOffsetResult =
-      getLogEndOffsets(Seq(topicPartition)).get(topicPartition).getOrElse(LogOffsetResult.Ignore)
+      getLogEndOffsets(Seq(topicPartition)).getOrElse(topicPartition, LogOffsetResult.Ignore)
 
     protected def getLogEndOffsets(topicPartitions: Seq[TopicPartition]): Map[TopicPartition, LogOffsetResult]
 
@@ -550,46 +548,43 @@ object ConsumerGroupCommand extends Logging {
     // `consumer` is only needed for `describe`, so we instantiate it lazily
     private var consumer: KafkaConsumer[String, String] = _
 
-    def listGroups(): List[String] = {
+    override def listGroups(): List[String] = {
       adminClient.listAllConsumerGroupsFlattened().map(_.groupId)
     }
 
-    def collectGroupOffsets(): (Option[String], Option[Seq[PartitionAssignmentState]]) = {
+    override def collectGroupOffsets(): (Option[String], Option[Seq[PartitionAssignmentState]]) = {
       val group = opts.options.valueOf(opts.groupOpt)
       val consumerGroupSummary = adminClient.describeConsumerGroup(group, opts.options.valueOf(opts.timeoutMsOpt))
-      (Some(consumerGroupSummary.state),
-        consumerGroupSummary.consumers match {
-          case None =>
-            None
-          case Some(consumers) =>
-            var assignedTopicPartitions = Array[TopicPartition]()
-            val offsets = adminClient.listGroupOffsets(group)
-            val rowsWithConsumer =
-              if (offsets.isEmpty)
-                List[PartitionAssignmentState]()
-              else {
-                consumers.filter(_.assignment.nonEmpty).sortWith(_.assignment.size > _.assignment.size).flatMap { consumerSummary =>
-                  val topicPartitions = consumerSummary.assignment
-                  assignedTopicPartitions = assignedTopicPartitions ++ consumerSummary.assignment
-                  val partitionOffsets: Map[TopicPartition, Option[Long]] = consumerSummary.assignment.map { topicPartition =>
-                    new TopicPartition(topicPartition.topic, topicPartition.partition) -> offsets.get(topicPartition)
-                  }.toMap
-                  collectConsumerAssignment(group, Some(consumerGroupSummary.coordinator), topicPartitions,
-                    partitionOffsets, Some(s"${consumerSummary.consumerId}"), Some(s"${consumerSummary.host}"),
-                    Some(s"${consumerSummary.clientId}"))
-                }
-              }
-
-            val rowsWithoutConsumer = offsets.filterKeys(!assignedTopicPartitions.contains(_)).flatMap {
-              case (topicPartition, offset) =>
-                collectConsumerAssignment(group, Some(consumerGroupSummary.coordinator), Seq(topicPartition),
-                    Map(topicPartition -> Some(offset)), Some(MISSING_COLUMN_VALUE),
-                    Some(MISSING_COLUMN_VALUE), Some(MISSING_COLUMN_VALUE))
+      val assignments = consumerGroupSummary.consumers.map { consumers =>
+        var assignedTopicPartitions = Array[TopicPartition]()
+        val offsets = adminClient.listGroupOffsets(group)
+        val rowsWithConsumer =
+          if (offsets.isEmpty)
+            List[PartitionAssignmentState]()
+          else {
+            consumers.filter(_.assignment.nonEmpty).sortWith(_.assignment.size > _.assignment.size).flatMap { consumerSummary =>
+              val topicPartitions = consumerSummary.assignment
+              assignedTopicPartitions = assignedTopicPartitions ++ consumerSummary.assignment
+              val partitionOffsets: Map[TopicPartition, Option[Long]] = consumerSummary.assignment.map { topicPartition =>
+                new TopicPartition(topicPartition.topic, topicPartition.partition) -> offsets.get(topicPartition)
+              }.toMap
+              collectConsumerAssignment(group, Some(consumerGroupSummary.coordinator), topicPartitions,
+                partitionOffsets, Some(s"${consumerSummary.consumerId}"), Some(s"${consumerSummary.host}"),
+                Some(s"${consumerSummary.clientId}"))
             }
+          }
 
-            Some(rowsWithConsumer ++ rowsWithoutConsumer)
+        val rowsWithoutConsumer = offsets.filterKeys(!assignedTopicPartitions.contains(_)).flatMap {
+          case (topicPartition, offset) =>
+            collectConsumerAssignment(group, Some(consumerGroupSummary.coordinator), Seq(topicPartition),
+              Map(topicPartition -> Some(offset)), Some(MISSING_COLUMN_VALUE),
+              Some(MISSING_COLUMN_VALUE), Some(MISSING_COLUMN_VALUE))
+        }
+
+        rowsWithConsumer ++ rowsWithoutConsumer
       }
-      )
+
+      (Some(consumerGroupSummary.state), assignments)
     }
 
     override def collectGroupMembers(verbose: Boolean): (Option[String], Option[Seq[MemberAssignmentState]]) = {
@@ -680,7 +675,9 @@ object ConsumerGroupCommand extends Logging {
         case "Empty" | "Dead" =>
           val partitionsToReset = getPartitionsToReset(groupId)
           val preparedOffsets = prepareOffsetsToReset(groupId, partitionsToReset)
-          val dryRun = opts.options.has(opts.dryRunOpt)
+
+          // Dry-run is the default behavior if --execute is not specified
+          val dryRun = opts.options.has(opts.dryRunOpt) || !opts.options.has(opts.executeOpt)
           if (!dryRun)
             getConsumer.commitSync(preparedOffsets.asJava)
           preparedOffsets
@@ -796,7 +793,7 @@ object ConsumerGroupCommand extends Logging {
         val (partitionsToResetWithCommittedOffset, partitionsToResetWithoutCommittedOffset) =
           partitionsToReset.partition(currentCommittedOffsets.keySet.contains(_))
 
-        val preparedOffsetsForParititionsWithCommittedOffset = partitionsToResetWithCommittedOffset.map { topicPartition =>
+        val preparedOffsetsForPartitionsWithCommittedOffset = partitionsToResetWithCommittedOffset.map { topicPartition =>
           (topicPartition, new OffsetAndMetadata(currentCommittedOffsets.get(topicPartition) match {
             case Some(offset) => offset
             case _ => throw new IllegalStateException(s"Expected a valid current offset for topic partition: $topicPartition")
@@ -808,7 +805,7 @@ object ConsumerGroupCommand extends Logging {
           case (topicPartition, _) => CommandLineUtils.printUsageAndDie(opts.parser, s"Error getting ending offset of topic partition: $topicPartition")
         }
 
-        preparedOffsetsForParititionsWithCommittedOffset ++ preparedOffsetsForPartitionsWithoutCommittedOffset
+        preparedOffsetsForPartitionsWithCommittedOffset ++ preparedOffsetsForPartitionsWithoutCommittedOffset
       } else {
         CommandLineUtils.printUsageAndDie(opts.parser, "Option '%s' requires one of the following scenarios: %s".format(opts.resetOffsetsOpt, opts.allResetOffsetScenarioOpts) )
       }
@@ -838,7 +835,7 @@ object ConsumerGroupCommand extends Logging {
     }
 
     override def exportOffsetsToReset(assignmentsToReset: Map[TopicPartition, OffsetAndMetadata]): String = {
-      val rows = assignmentsToReset.map { case (k,v) => s"${k.topic()},${k.partition()},${v.offset()}" }(collection.breakOut): List[String]
+      val rows = assignmentsToReset.map { case (k,v) => s"${k.topic},${k.partition},${v.offset}" }(collection.breakOut): List[String]
       rows.foldRight("")(_ + "\n" + _)
     }
 
@@ -899,10 +896,13 @@ object ConsumerGroupCommand extends Logging {
       "or is going through some changes)."
     val CommandConfigDoc = "Property file containing configs to be passed to Admin Client and Consumer."
     val ResetOffsetsDoc = "Reset offsets of consumer group. Supports one consumer group at the time, and instances should be inactive" + nl +
-      "Has 3 execution options: (default) to plan which offsets to reset, --execute to execute the reset-offsets process, and --export to export the results to a CSV format." + nl +
-      "Has the following scenarios to choose: --to-datetime, --by-period, --to-earliest, --to-latest, --shift-by, --from-file, --to-current. One scenario must be choose" + nl +
-      "To define the scope use: --all-topics or --topic. . One scope must be choose, unless you use '--from-file' scenario"
+      "Has 2 execution options: --dry-run (the default) to plan which offsets to reset, and --execute to update the offsets. " +
+      "Additionally, the --export option is used to export the results to a CSV format." + nl +
+      "You must choose one of the following reset specifications: --to-datetime, --by-period, --to-earliest, " +
+      "--to-latest, --shift-by, --from-file, --to-current." + nl +
+      "To define the scope use --all-topics or --topic. One scope must be specified unless you use '--from-file'."
     val DryRunDoc = "Only show results without executing changes on Consumer Groups. Supported operations: reset-offsets."
+    val ExecuteDoc = "Execute operation. Supported operations: reset-offsets."
     val ExportDoc = "Export operation execution to a CSV file. Supported operations: reset-offsets."
     val ResetToOffsetDoc = "Reset offsets to a specific offset."
     val ResetFromFileDoc = "Reset offsets to values defined in CSV file."
@@ -955,6 +955,7 @@ object ConsumerGroupCommand extends Logging {
                                   .ofType(classOf[String])
     val resetOffsetsOpt = parser.accepts("reset-offsets", ResetOffsetsDoc)
     val dryRunOpt = parser.accepts("dry-run", DryRunDoc)
+    val executeOpt = parser.accepts("execute", ExecuteDoc)
     val exportOpt = parser.accepts("export", ExportDoc)
     val resetToOffsetOpt = parser.accepts("to-offset", ResetToOffsetDoc)
                            .withRequiredArg()
@@ -1029,8 +1030,19 @@ object ConsumerGroupCommand extends Logging {
         CommandLineUtils.checkRequiredArgs(parser, options, groupOpt)
 
       if (options.has(deleteOpt) && !options.has(groupOpt) && !options.has(topicOpt))
-        CommandLineUtils.printUsageAndDie(parser, "Option %s either takes %s, %s, or both".format(deleteOpt, groupOpt, topicOpt))
-      if (options.has(resetOffsetsOpt))
+        CommandLineUtils.printUsageAndDie(parser, s"Option $deleteOpt either takes $groupOpt, $topicOpt, or both")
+
+      if (options.has(resetOffsetsOpt)) {
+        if (!options.has(dryRunOpt) && !options.has(executeOpt)) {
+          Console.err.println("WARN: In a future major release, the default behavior of this command will be to " +
+            "prompt the user before executing the reset rather than doing a dry run. You should add the --dry-run " +
+            "option explicitly if you are scripting this command and want to keep the current default behavior " +
+            "without prompting.")
+        }
+
+        if (options.has(dryRunOpt) && options.has(executeOpt))
+          CommandLineUtils.printUsageAndDie(parser, s"Option $resetOffsetsOpt only accepts one of $executeOpt and $dryRunOpt")
+
         CommandLineUtils.checkRequiredArgs(parser, options, groupOpt)
         CommandLineUtils.checkInvalidArgs(parser, options, resetToOffsetOpt, allResetOffsetScenarioOpts - resetToOffsetOpt)
         CommandLineUtils.checkInvalidArgs(parser, options, resetToDatetimeOpt, allResetOffsetScenarioOpts - resetToDatetimeOpt)
@@ -1040,7 +1052,7 @@ object ConsumerGroupCommand extends Logging {
         CommandLineUtils.checkInvalidArgs(parser, options, resetToCurrentOpt, allResetOffsetScenarioOpts - resetToCurrentOpt)
         CommandLineUtils.checkInvalidArgs(parser, options, resetShiftByOpt, allResetOffsetScenarioOpts - resetShiftByOpt)
         CommandLineUtils.checkInvalidArgs(parser, options, resetFromFileOpt, allResetOffsetScenarioOpts - resetFromFileOpt)
-
+      }
 
       // check invalid args
       CommandLineUtils.checkInvalidArgs(parser, options, groupOpt, allConsumerGroupLevelOpts - describeOpt - deleteOpt - resetOffsetsOpt)
diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java
index 4496876e09a78..31c69eedc1d33 100644
--- a/core/src/main/scala/kafka/tools/StreamsResetter.java
+++ b/core/src/main/scala/kafka/tools/StreamsResetter.java
@@ -91,6 +91,7 @@ public class StreamsResetter {
     private static OptionSpec fromFileOption;
     private static OptionSpec shiftByOption;
     private static OptionSpecBuilder dryRunOption;
+    private static OptionSpecBuilder executeOption;
     private static OptionSpec commandConfigOption;
 
     private OptionSet options = null;
@@ -109,6 +110,7 @@ public int run(final String[] args,
 
         try {
             parseArguments(args);
+
             final boolean dryRun = options.has(dryRunOption);
 
             final String groupId = options.valueOf(applicationIdOption);
@@ -207,6 +209,7 @@ private void parseArguments(final String[] args) throws IOException {
             .withRequiredArg()
             .ofType(String.class)
             .describedAs("file name");
+        executeOption = optionParser.accepts("execute", "Execute the command.");
         dryRunOption = optionParser.accepts("dry-run", "Display the actions that would be performed without executing the reset commands.");
 
         // TODO: deprecated in 1.0; can be removed eventually
@@ -219,6 +222,16 @@ private void parseArguments(final String[] args) throws IOException {
             throw e;
         }
 
+        if (options.has(executeOption) && options.has(dryRunOption)) {
+            CommandLineUtils.printUsageAndDie(optionParser, "Only one of --dry-run and --execute can be specified");
+        }
+
+        if (!options.has(executeOption) && !options.has(dryRunOption)) {
+            System.err.println("WARN: In a future major release, the default behavior of this command will be to " +
+                    "prompt the user before executing the reset. You should add the --execute option explicitly if " +
+                    "you are scripting this command and want to keep the current default behavior without prompting.");
+        }
+
         scala.collection.immutable.HashSet> allScenarioOptions = new scala.collection.immutable.HashSet<>();
         allScenarioOptions.$plus(toOffsetOption);
         allScenarioOptions.$plus(toDatetimeOption);
@@ -266,7 +279,6 @@ private int maybeResetInputAndSeekToEndIntermediateTopicOffsets(final Map consum
                 notFoundInputTopics.add(topic);
             } else {
                 topicsToSubscribe.add(topic);
-
             }
         }
         for (final String topic : intermediateTopics) {
@@ -277,6 +289,28 @@ private int maybeResetInputAndSeekToEndIntermediateTopicOffsets(final Map consum
             }
         }
 
+        if (!notFoundInputTopics.isEmpty()) {
+            System.out.println("Following input topics are not found, skipping them");
+            for (final String topic : notFoundInputTopics) {
+                System.out.println("Topic: " + topic);
+            }
+            topicNotFound = EXIT_CODE_ERROR;
+        }
+
+        if (!notFoundIntermediateTopics.isEmpty()) {
+            System.out.println("Following intermediate topics are not found, skipping them");
+            for (final String topic : notFoundIntermediateTopics) {
+                System.out.println("Topic:" + topic);
+            }
+            topicNotFound = EXIT_CODE_ERROR;
+        }
+
+        // Return early if there are no topics to reset (the consumer will raise an error if we
+        // try to poll with an empty subscription)
+        if (topicsToSubscribe.isEmpty()) {
+            return topicNotFound;
+        }
+
         final Properties config = new Properties();
         config.putAll(consumerConfig);
         config.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
@@ -311,22 +345,6 @@ private int maybeResetInputAndSeekToEndIntermediateTopicOffsets(final Map consum
                 }
                 client.commitSync();
             }
-
-            if (notFoundInputTopics.size() > 0) {
-                System.out.println("Following input topics are not found, skipping them");
-                for (final String topic : notFoundInputTopics) {
-                    System.out.println("Topic: " + topic);
-                }
-                topicNotFound = EXIT_CODE_ERROR;
-            }
-
-            if (notFoundIntermediateTopics.size() > 0) {
-                System.out.println("Following intermediate topics are not found, skipping them");
-                for (final String topic : notFoundIntermediateTopics) {
-                    System.out.println("Topic:" + topic);
-                }
-            }
-
         } catch (final Exception e) {
             System.err.println("ERROR: Resetting offsets failed.");
             throw e;
@@ -337,8 +355,8 @@ private int maybeResetInputAndSeekToEndIntermediateTopicOffsets(final Map consum
 
     // visible for testing
     public void maybeSeekToEnd(final String groupId,
-                                final Consumer client,
-                                final Set intermediateTopicPartitions) {
+                               final Consumer client,
+                               final Set intermediateTopicPartitions) {
         if (intermediateTopicPartitions.size() > 0) {
             System.out.println("Following intermediate topics offsets will be reset to end (for consumer group " + groupId + ")");
             for (final TopicPartition topicPartition : intermediateTopicPartitions) {
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
index 4ca04b02f4e93..7c451093e676a 100644
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
@@ -74,6 +74,25 @@ class ConsumerGroupCommandTest extends KafkaServerTestHarness {
     oldConsumers += new OldConsumer(Whitelist(topic), consumerProps)
   }
 
+  def committedOffsets(topic: String = topic, group: String = group): Map[TopicPartition, Long] = {
+    val props = new Properties
+    props.put("bootstrap.servers", brokerList)
+    props.put("group.id", group)
+    val consumer = new KafkaConsumer(props, new StringDeserializer, new StringDeserializer)
+    try {
+      consumer.partitionsFor(topic).asScala.flatMap { partitionInfo =>
+        val tp = new TopicPartition(partitionInfo.topic, partitionInfo.partition)
+        val committed = consumer.committed(tp)
+        if (committed == null)
+          None
+        else
+          Some(tp -> committed.offset)
+      }.toMap
+    } finally {
+      consumer.close()
+    }
+  }
+
   def stopRandomOldConsumer(): Unit = {
     oldConsumers.head.stop()
   }
diff --git a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
index befe65c41100f..3d8e895598237 100644
--- a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
@@ -16,9 +16,10 @@ import java.io.{BufferedWriter, File, FileWriter}
 import java.text.{ParseException, SimpleDateFormat}
 import java.util.{Calendar, Date, Properties}
 
-import kafka.admin.ConsumerGroupCommand.{ConsumerGroupCommandOptions, ConsumerGroupService, KafkaConsumerGroupService}
+import kafka.admin.ConsumerGroupCommand.ConsumerGroupService
 import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
+import org.apache.kafka.common.TopicPartition
 import org.junit.Assert._
 import org.junit.Test
 
@@ -73,10 +74,6 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
   val topic1 = "foo1"
   val topic2 = "foo2"
 
-  /**
-    * Implementations must override this method to return a set of KafkaConfigs. This method will be invoked for every
-    * test and should not reuse previous configurations unless they select their ports randomly when servers are started.
-    */
   override def generateConfigs: Seq[KafkaConfig] = {
     TestUtils.createBrokerConfigs(1, zkConnect, enableControlledShutdown = false)
       .map(KafkaConfig.fromProps(_, overridingProps))  
@@ -84,469 +81,300 @@ class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
 
   @Test
   def testResetOffsetsNotExistingGroup() {
-    addConsumerGroupExecutor(numConsumers = 1, topic1)
-
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", "missing.group", "--all-topics", "--to-current")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset == Map.empty
-    }, "Expected to have an empty assignations map.")
-
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", "missing.group", "--all-topics",
+      "--to-current", "--execute")
+    val consumerGroupCommand = getConsumerGroupService(args)
+    val resetOffsets = consumerGroupCommand.resetOffsets()
+    assertEquals(Map.empty, resetOffsets)
+    assertEquals(resetOffsets, committedOffsets(group = "missing.group"))
   }
 
   @Test
   def testResetOffsetsNewConsumerExistingTopic(): Unit = {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", "new.group", "--topic", topic1, "--to-offset", "50")
-    val opts = new ConsumerGroupCommandOptions(cgcArgs)
-    val consumerGroupCommand = new KafkaConsumerGroupService(opts)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 50 })
-    }, "Expected the consumer group to reset to offset 1 (specific offset).")
-
-    printConsumerGroup("new.group")
-    adminZkClient.deleteTopic(topic1)
-    consumerGroupCommand.close()
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", "new.group", "--topic", topic,
+      "--to-offset", "50")
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true)
+    resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true)
+    resetAndAssertOffsets(args ++ Array("--execute"), expectedOffset = 50, group = "new.group")
   }
 
   @Test
   def testResetOffsetsToLocalDateTime() {
-    adminZkClient.createTopic(topic1, 1, 1)
-
     val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
     val calendar = Calendar.getInstance()
     calendar.add(Calendar.DATE, -1)
 
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--dry-run")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    val executor = addConsumerGroupExecutor(numConsumers = 1, topic1)
-
-    TestUtils.waitUntilTrue(() => {
-      val (_, assignmentsOption) = consumerGroupCommand.collectGroupOffsets()
-      assignmentsOption match {
-        case Some(assignments) =>
-          val sumOffset = assignments.filter(_.topic.exists(_ == topic1))
-            .filter(_.offset.isDefined)
-            .map(assignment => assignment.offset.get)
-            .foldLeft(0.toLong)(_ + _)
-          sumOffset == 100
-        case _ => false
-      }
-    }, "Expected that consumer group has consumed all messages from topic/partition.")
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
 
+    val executor = addConsumerGroupExecutor(numConsumers = 1, topic)
+    awaitConsumerProgress(count = 100L)
     executor.shutdown()
 
-    val cgcArgs1 = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-datetime", format.format(calendar.getTime))
-    val consumerGroupCommand1 = getConsumerGroupService(cgcArgs1)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand1.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 }
-    }, "Expected the consumer group to reset to when offset was 50.")
-
-    printConsumerGroup()
-
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-datetime", format.format(calendar.getTime), "--execute")
+    resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsToZonedDateTime() {
-    adminZkClient.createTopic(topic1, 1, 1)
-    TestUtils.produceMessages(servers, topic1, 50, acks = 1, 100 * 1000)
-
     val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
-    val checkpoint = new Date()
-
-    TestUtils.produceMessages(servers, topic1, 50, acks = 1, 100 * 1000)
 
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--dry-run")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    val executor = addConsumerGroupExecutor(numConsumers = 1, topic1)
-
-    TestUtils.waitUntilTrue(() => {
-      val (_, assignmentsOption) = consumerGroupCommand.collectGroupOffsets()
-      assignmentsOption match {
-        case Some(assignments) =>
-          val sumOffset = (assignments.filter(_.topic.exists(_ == topic1))
-            .filter(_.offset.isDefined)
-            .map(assignment => assignment.offset.get) foldLeft 0.toLong)(_ + _)
-          sumOffset == 100
-        case _ => false
-      }
-    }, "Expected that consumer group has consumed all messages from topic/partition.")
+    TestUtils.produceMessages(servers, topic, 50, acks = 1, 100 * 1000)
+    val checkpoint = new Date()
+    TestUtils.produceMessages(servers, topic, 50, acks = 1, 100 * 1000)
 
+    val executor = addConsumerGroupExecutor(numConsumers = 1, topic)
+    awaitConsumerProgress(count = 100L)
     executor.shutdown()
 
-    val cgcArgs1 = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-datetime", format.format(checkpoint))
-    val consumerGroupCommand1 = getConsumerGroupService(cgcArgs1)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand1.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 50 }
-    }, "Expected the consumer group to reset to when offset was 50.")
-
-    printConsumerGroup()
-
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-datetime", format.format(checkpoint), "--execute")
+    resetAndAssertOffsets(args, expectedOffset = 50)
   }
 
   @Test
   def testResetOffsetsByDuration() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--by-duration", "PT1M")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.waitUntilTrue(() => {
-        val assignmentsToReset = consumerGroupCommand.resetOffsets()
-        assignmentsToReset.exists { assignment => assignment._2.offset() == 0 }
-    }, "Expected the consumer group to reset to offset 0 (earliest by duration).")
-
-    printConsumerGroup()
-
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--by-duration", "PT1M", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsByDurationToEarliest() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--by-duration", "PT0.1S")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 100 }
-    }, "Expected the consumer group to reset to offset 100 (latest by duration).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--by-duration", "PT0.1S", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    resetAndAssertOffsets(args, expectedOffset = 100)
   }
 
   @Test
   def testResetOffsetsToEarliest() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-earliest")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 }
-    }, "Expected the consumer group to reset to offset 0 (earliest).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-earliest", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsToLatest() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-latest")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 200 })
-    }, "Expected the consumer group to reset to offset 200 (latest).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-latest", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 200)
   }
 
   @Test
   def testResetOffsetsToCurrentOffset() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-current")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 100 })
-    }, "Expected the consumer group to reset to offset 100 (current).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
-  }
-
-  private def produceConsumeAndShutdown(consumerGroupCommand: ConsumerGroupService, numConsumers: Int, topic: String, totalMessages: Int) {
-    TestUtils.produceMessages(servers, topic, totalMessages, acks = 1, 100 * 1000)
-    val executor =  addConsumerGroupExecutor(numConsumers, topic)
-
-    TestUtils.waitUntilTrue(() => {
-      val (_, assignmentsOption) = consumerGroupCommand.collectGroupOffsets()
-      assignmentsOption match {
-        case Some(assignments) =>
-          val sumOffset = assignments.filter(_.topic.exists(_ == topic))
-            .filter(_.offset.isDefined)
-            .map(assignment => assignment.offset.get)
-            .foldLeft(0.toLong)(_ + _)
-          sumOffset == totalMessages
-        case _ => false
-      }
-    }, "Expected the consumer group to consume all messages from topic.")
-
-    executor.shutdown()
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-current", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 100)
   }
 
   @Test
   def testResetOffsetsToSpecificOffset() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-offset", "1")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 1 })
-    }, "Expected the consumer group to reset to offset 1 (specific offset).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-offset", "1", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    resetAndAssertOffsets(args, expectedOffset = 1)
   }
 
   @Test
   def testResetOffsetsShiftPlus() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--shift-by", "50")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 150 })
-    }, "Expected the consumer group to reset to offset 150 (current + 50).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--shift-by", "50", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 150)
   }
 
   @Test
   def testResetOffsetsShiftMinus() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--shift-by", "-50")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 50 })
-    }, "Expected the consumer group to reset to offset 50 (current - 50).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--shift-by", "-50", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 50)
   }
 
   @Test
   def testResetOffsetsShiftByLowerThanEarliest() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--shift-by", "-150")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 0 })
-    }, "Expected the consumer group to reset to offset 0 (earliest by shift).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--shift-by", "-150", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsShiftByHigherThanLatest() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--shift-by", "150")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.produceMessages(servers, topic1, 100, acks = 1, 100 * 1000)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists({ assignment => assignment._2.offset() == 200 })
-    }, "Expected the consumer group to reset to offset 200 (latest by shift).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--shift-by", "150", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    TestUtils.produceMessages(servers, topic, 100, acks = 1, 100 * 1000)
+    resetAndAssertOffsets(args, expectedOffset = 200)
   }
 
   @Test
   def testResetOffsetsToEarliestOnOneTopic() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", topic1, "--to-earliest")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
-    adminZkClient.createTopic(topic1, 1, 1)
-
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 }
-    }, "Expected the consumer group to reset to offset 0 (earliest).")
-
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", topic,
+      "--to-earliest", "--execute")
+    produceConsumeAndShutdown(totalMessages = 100)
+    resetAndAssertOffsets(args, expectedOffset = 0)
   }
 
   @Test
   def testResetOffsetsToEarliestOnOneTopicAndPartition() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", String.format("%s:1", topic1), "--to-earliest")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
+    val topic = "bar"
+    adminZkClient.createTopic(topic, 2, 1)
 
-    adminZkClient.createTopic(topic1, 2, 1)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic",
+      s"$topic:1", "--to-earliest", "--execute")
+    val consumerGroupCommand = getConsumerGroupService(args)
 
-    produceConsumeAndShutdown(consumerGroupCommand, 2, topic1, 100)
+    produceConsumeAndShutdown(totalMessages = 100, numConsumers = 2, topic)
+    val priorCommittedOffsets = committedOffsets(topic = topic)
 
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 && assignment._1.partition() == 1 }
-    }, "Expected the consumer group to reset to offset 0 (earliest) in partition 1.")
+    val tp0 = new TopicPartition(topic, 0)
+    val tp1 = new TopicPartition(topic, 1)
+    val expectedOffsets = Map(tp0 -> priorCommittedOffsets(tp0), tp1 -> 0L)
+    resetAndAssertOffsetsCommitted(consumerGroupCommand, expectedOffsets, topic)
 
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+    adminZkClient.deleteTopic(topic)
   }
 
   @Test
   def testResetOffsetsToEarliestOnTopics() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets",
-      "--group", group,
-      "--topic", topic1,
-      "--topic", topic2,
-      "--to-earliest")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
+    val topic1 = "topic1"
+    val topic2 = "topic2"
     adminZkClient.createTopic(topic1, 1, 1)
     adminZkClient.createTopic(topic2, 1, 1)
 
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic1, 100)
-    produceConsumeAndShutdown(consumerGroupCommand, 1, topic2, 100)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic", topic1,
+      "--topic", topic2, "--to-earliest", "--execute")
+    val consumerGroupCommand = getConsumerGroupService(args)
 
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 && assignment._1.topic() == topic1 } &&
-        assignmentsToReset.exists { assignment => assignment._2.offset() == 0 && assignment._1.topic() == topic2 }
-    }, "Expected the consumer group to reset to offset 0 (earliest).")
+    produceConsumeAndShutdown(100, 1, topic1)
+    produceConsumeAndShutdown(100, 1, topic2)
+
+    val tp1 = new TopicPartition(topic1, 0)
+    val tp2 = new TopicPartition(topic2, 0)
+
+    val allResetOffsets = resetOffsets(consumerGroupCommand)
+    assertEquals(Map(tp1 -> 0L, tp2 -> 0L), allResetOffsets)
+    assertEquals(Map(tp1 -> 0L), committedOffsets(topic1))
+    assertEquals(Map(tp2 -> 0L), committedOffsets(topic2))
 
-    printConsumerGroup()
     adminZkClient.deleteTopic(topic1)
     adminZkClient.deleteTopic(topic2)
   }
 
   @Test
   def testResetOffsetsToEarliestOnTopicsAndPartitions() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets",
-      "--group", group,
-      "--topic", String.format("%s:1", topic1),
-      "--topic", String.format("%s:1", topic2),
-      "--to-earliest")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
+    val topic1 = "topic1"
+    val topic2 = "topic2"
 
     adminZkClient.createTopic(topic1, 2, 1)
     adminZkClient.createTopic(topic2, 2, 1)
 
-    produceConsumeAndShutdown(consumerGroupCommand, 2, topic1, 100)
-    produceConsumeAndShutdown(consumerGroupCommand, 2, topic2, 100)
+    val args = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--topic",
+      s"$topic1:1", "--topic", s"$topic2:1", "--to-earliest", "--execute")
+    val consumerGroupCommand = getConsumerGroupService(args)
 
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 && assignment._1.partition() == 1 && assignment._1.topic() == topic1 }
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 0 && assignment._1.partition() == 1 && assignment._1.topic() == topic2 }
-    }, "Expected the consumer group to reset to offset 0 (earliest) in partition 1.")
+    produceConsumeAndShutdown(100, 2, topic1)
+    produceConsumeAndShutdown(100, 2, topic2)
+
+    val priorCommittedOffsets1 = committedOffsets(topic1)
+    val priorCommittedOffsets2 = committedOffsets(topic2)
+
+    val tp1 = new TopicPartition(topic1, 1)
+    val tp2 = new TopicPartition(topic2, 1)
+    val allResetOffsets = resetOffsets(consumerGroupCommand)
+    assertEquals(Map(tp1 -> 0, tp2 -> 0), allResetOffsets)
+
+    assertEquals(priorCommittedOffsets1 + (tp1 -> 0L), committedOffsets(topic1))
+    assertEquals(priorCommittedOffsets2 + (tp2 -> 0L), committedOffsets(topic2))
 
-    printConsumerGroup()
     adminZkClient.deleteTopic(topic1)
     adminZkClient.deleteTopic(topic2)
   }
 
   @Test
   def testResetOffsetsExportImportPlan() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--to-offset","2", "--export")
-    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
+    val topic = "bar"
+    val tp0 = new TopicPartition(topic, 0)
+    val tp1 = new TopicPartition(topic, 1)
+    adminZkClient.createTopic(topic, 2, 1)
 
-    adminZkClient.createTopic(topic1, 2, 1)
+    val cgcArgs = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--to-offset", "2", "--export")
+    val consumerGroupCommand = getConsumerGroupService(cgcArgs)
 
-    produceConsumeAndShutdown(consumerGroupCommand, 2, topic1, 100)
+    produceConsumeAndShutdown(100, 2, topic)
 
     val file = File.createTempFile("reset", ".csv")
+    file.deleteOnExit()
 
-    TestUtils.waitUntilTrue(() => {
-      val assignmentsToReset = consumerGroupCommand.resetOffsets()
-      val bw = new BufferedWriter(new FileWriter(file))
-      bw.write(consumerGroupCommand.exportOffsetsToReset(assignmentsToReset))
-      bw.close()
-      assignmentsToReset.exists { assignment => assignment._2.offset() == 2 } && file.exists()
-    }, "Expected the consume all messages and save reset offsets plan to file")
-
+    val exportedOffsets = consumerGroupCommand.resetOffsets()
+    val bw = new BufferedWriter(new FileWriter(file))
+    bw.write(consumerGroupCommand.exportOffsetsToReset(exportedOffsets))
+    bw.close()
+    assertEquals(Map(tp0 -> 2L, tp1 -> 2L), exportedOffsets.mapValues(_.offset))
 
-    val cgcArgsExec = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
+    val cgcArgsExec = Array("--bootstrap-server", brokerList, "--reset-offsets", "--group", group, "--all-topics",
+      "--from-file", file.getCanonicalPath, "--dry-run")
     val consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec)
+    val importedOffsets = consumerGroupCommandExec.resetOffsets()
+    assertEquals(Map(tp0 -> 2L, tp1 -> 2L), importedOffsets.mapValues(_.offset))
 
-    TestUtils.waitUntilTrue(() => {
-        val assignmentsToReset = consumerGroupCommandExec.resetOffsets()
-        assignmentsToReset.exists { assignment => assignment._2.offset() == 2 }
-    }, "Expected the consumer group to reset to offset 2 according to the plan in the file.")
+    adminZkClient.deleteTopic(topic)
+  }
 
-    file.deleteOnExit()
+  private def produceConsumeAndShutdown(totalMessages: Int, numConsumers: Int = 1, topic: String = topic) {
+    TestUtils.produceMessages(servers, topic, totalMessages, acks = 1, 100 * 1000)
+    val executor =  addConsumerGroupExecutor(numConsumers, topic)
+    awaitConsumerProgress(topic, totalMessages)
+    executor.shutdown()
+  }
 
-    printConsumerGroup()
-    adminZkClient.deleteTopic(topic1)
+  private def awaitConsumerProgress(topic: String = topic, count: Long): Unit = {
+    TestUtils.waitUntilTrue(() => {
+      val offsets = committedOffsets(topic).values
+      count == offsets.sum
+    }, "Expected that consumer group has consumed all messages from topic/partition.")
   }
 
-  private def printConsumerGroup() {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--group", group, "--describe")
-    ConsumerGroupCommand.main(cgcArgs)
+  private def resetAndAssertOffsets(args: Array[String],
+                                           expectedOffset: Long,
+                                           group: String = group,
+                                           dryRun: Boolean = false): Unit = {
+    val consumerGroupCommand = getConsumerGroupService(args)
+    try {
+      val priorOffsets = committedOffsets(group = group)
+      val expectedOffsets = Map(new TopicPartition(topic, 0) -> expectedOffset)
+      assertEquals(expectedOffsets, resetOffsets(consumerGroupCommand))
+      assertEquals(if (dryRun) priorOffsets else expectedOffsets, committedOffsets(group = group))
+    } finally {
+      consumerGroupCommand.close()
+    }
+  }
+
+  private def resetAndAssertOffsetsCommitted(consumerGroupService: ConsumerGroupService,
+                                             expectedOffsets: Map[TopicPartition, Long],
+                                             topic: String = topic): Unit = {
+    val allResetOffsets = resetOffsets(consumerGroupService)
+    allResetOffsets.foreach { case (tp, offset) =>
+      assertEquals(offset, expectedOffsets(tp))
+    }
+    assertEquals(expectedOffsets, committedOffsets(topic))
   }
 
-  private def printConsumerGroup(group: String) {
-    val cgcArgs = Array("--bootstrap-server", brokerList, "--group", group, "--describe")
-    ConsumerGroupCommand.main(cgcArgs)
+  private def resetOffsets(consumerGroupService: ConsumerGroupService): Map[TopicPartition, Long] = {
+    consumerGroupService.resetOffsets().mapValues(_.offset)
   }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
index 758c4f6a8bcf0..8a82bf9fe5eb5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/AbstractResetIntegrationTest.java
@@ -219,7 +219,9 @@ void shouldNotAllowToResetWhileStreamsIsRunning() throws Exception {
         final String[] parameters = new String[] {
             "--application-id", appID,
             "--bootstrap-servers", cluster.bootstrapServers(),
-            "--input-topics", NON_EXISTING_TOPIC };
+            "--input-topics", NON_EXISTING_TOPIC,
+            "--execute"
+        };
         final Properties cleanUpConfig = new Properties();
         cleanUpConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 100);
         cleanUpConfig.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "" + CLEANUP_CONSUMER_TIMEOUT);
@@ -241,7 +243,9 @@ public void shouldNotAllowToResetWhenInputTopicAbsent() throws Exception {
         final String[] parameters = new String[] {
             "--application-id", appID,
             "--bootstrap-servers", cluster.bootstrapServers(),
-            "--input-topics", NON_EXISTING_TOPIC };
+            "--input-topics", NON_EXISTING_TOPIC,
+            "--execute"
+        };
         final Properties cleanUpConfig = new Properties();
         cleanUpConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 100);
         cleanUpConfig.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "" + CLEANUP_CONSUMER_TIMEOUT);
@@ -255,7 +259,9 @@ public void shouldNotAllowToResetWhenIntermediateTopicAbsent() throws Exception
         final String[] parameters = new String[] {
             "--application-id", appID,
             "--bootstrap-servers", cluster.bootstrapServers(),
-            "--input-topics", NON_EXISTING_TOPIC };
+            "--intermediate-topics", NON_EXISTING_TOPIC,
+            "--execute"
+        };
         final Properties cleanUpConfig = new Properties();
         cleanUpConfig.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 100);
         cleanUpConfig.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "" + CLEANUP_CONSUMER_TIMEOUT);
@@ -548,8 +554,9 @@ private void cleanGlobal(final boolean withIntermediateTopics,
         // leaving --zookeeper arg here to ensure tool works if users add it
         final List parameterList = new ArrayList<>(
             Arrays.asList("--application-id", appID,
-                "--bootstrap-servers", cluster.bootstrapServers(),
-                "--input-topics", INPUT_TOPIC));
+                    "--bootstrap-servers", cluster.bootstrapServers(),
+                    "--input-topics", INPUT_TOPIC,
+                    "--execute"));
         if (withIntermediateTopics) {
             parameterList.add("--intermediate-topics");
             parameterList.add(INTERMEDIATE_USER_TOPIC);
diff --git a/streams/src/test/java/org/apache/kafka/streams/tools/StreamsResetterTest.java b/streams/src/test/java/org/apache/kafka/streams/tools/StreamsResetterTest.java
index dd32ad0786a8c..ad19f32fd1d74 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tools/StreamsResetterTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tools/StreamsResetterTest.java
@@ -43,9 +43,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
-/**
- *
- */
 public class StreamsResetterTest {
 
     private static final String TOPIC = "topic1";

From 0cd83e997dd8ca3ac0cc6458eee75fdf7b66ae57 Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Fri, 23 Feb 2018 16:13:57 -0800
Subject: [PATCH 0096/1847] MINOR: wait for broker startup for system tests
 (#4363)

ensure that brokers are registered at ZK before start() returns

Author: Matthias J. Sax 

Reviewers: Ewen Cheslack-Postava , Damian Guy , Guozhang Wang 
---
 tests/kafkatest/services/kafka/kafka.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index 30dcdb59c36ce..4a35e3f26aecd 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -165,6 +165,15 @@ def start(self, add_principals=""):
         self.start_minikdc(add_principals)
         Service.start(self)
 
+        self.logger.info("Waiting for brokers to register at ZK")
+
+        retries = 30
+        expected_broker_ids = set(self.nodes)
+        wait_until(lambda: {node for node in self.nodes if self.is_registered(node)} == expected_broker_ids, 30, 1)
+
+        if retries == 0:
+            raise RuntimeError("Kafka servers didn't register at ZK within 30 seconds")
+
         # Create topics if necessary
         if self.topics is not None:
             for topic, topic_cfg in self.topics.items():

From 4a027a0af10bc10bb223856100fdc19c6186a405 Mon Sep 17 00:00:00 2001
From: Daniel Shuy 
Date: Sun, 25 Feb 2018 04:07:42 +0800
Subject: [PATCH 0097/1847] Fix typo in MockProducer JavaDoc (#4606)

---
 .../java/org/apache/kafka/clients/producer/MockProducer.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
index d2a84c66abeea..ddf5b88720526 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
@@ -77,7 +77,7 @@ public class MockProducer implements Producer {
      * @param cluster The cluster holding metadata for this producer
      * @param autoComplete If true automatically complete all requests successfully and execute the callback. Otherwise
      *        the user must call {@link #completeNext()} or {@link #errorNext(RuntimeException)} after
-     *        {@link #send(ProducerRecord) send()} to complete the call and unblock the @{link
+     *        {@link #send(ProducerRecord) send()} to complete the call and unblock the {@link
      *        java.util.concurrent.Future Future<RecordMetadata>} that is returned.
      * @param partitioner The partition strategy
      * @param keySerializer The serializer for key that implements {@link Serializer}.

From 99d650c2c882cc1f359a1dbb5fc6bea2d5b303b6 Mon Sep 17 00:00:00 2001
From: Igor Kostiakov 
Date: Sun, 25 Feb 2018 03:43:10 +0100
Subject: [PATCH 0098/1847] KAFKA-6590; Fix bug in aggregation of consumer
 fetch bytes and counts metrics (#4278)

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../clients/consumer/internals/Fetcher.java   |  4 +-
 .../consumer/internals/FetcherTest.java       | 54 +++++++++++++------
 2 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index ac199bc4b0db6..b46a3a29380c9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -1276,8 +1276,8 @@ public void record(TopicPartition partition, int bytes, int records) {
 
             if (this.unrecordedPartitions.isEmpty()) {
                 // once all expected partitions from the fetch have reported in, record the metrics
-                this.sensors.bytesFetched.record(topicFetchMetric.fetchBytes);
-                this.sensors.recordsFetched.record(topicFetchMetric.fetchRecords);
+                this.sensors.bytesFetched.record(this.fetchMetrics.fetchBytes);
+                this.sensors.recordsFetched.record(this.fetchMetrics.fetchRecords);
 
                 // also record per-topic metrics
                 for (Map.Entry entry: this.topicFetchMetrics.entrySet()) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index e8bb4e696ba21..27aba41074622 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -1447,26 +1447,50 @@ public void testReadCommittedLagMetric() {
 
     @Test
     public void testFetchResponseMetrics() {
-        subscriptions.assignFromUser(singleton(tp0));
-        subscriptions.seek(tp0, 0);
+        String topic1 = "foo";
+        String topic2 = "bar";
+        TopicPartition tp1 = new TopicPartition(topic1, 0);
+        TopicPartition tp2 = new TopicPartition(topic2, 0);
+
+        Map partitionCounts = new HashMap<>();
+        partitionCounts.put(topic1, 1);
+        partitionCounts.put(topic2, 1);
+        Cluster cluster = TestUtils.clusterWith(1, partitionCounts);
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
 
-        Map allMetrics = metrics.metrics();
-        KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
-        KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
-
-        MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE,
-                TimestampType.CREATE_TIME, 0L);
-        for (int v = 0; v < 3; v++)
-            builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
-        MemoryRecords records = builder.build();
+        subscriptions.assignFromUser(Utils.mkSet(tp1, tp2));
 
         int expectedBytes = 0;
-        for (Record record : records.records())
-            expectedBytes += record.sizeInBytes();
+        LinkedHashMap fetchPartitionData = new LinkedHashMap<>();
 
-        fetchRecords(tp0, records, Errors.NONE, 100L, 0);
+        for (TopicPartition tp : Utils.mkSet(tp1, tp2)) {
+            subscriptions.seek(tp, 0);
+
+            MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE,
+                    TimestampType.CREATE_TIME, 0L);
+            for (int v = 0; v < 3; v++)
+                builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
+            MemoryRecords records = builder.build();
+            for (Record record : records.records())
+                expectedBytes += record.sizeInBytes();
+
+            fetchPartitionData.put(tp, new FetchResponse.PartitionData(Errors.NONE, 15L,
+                    FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records));
+        }
+
+        assertEquals(1, fetcher.sendFetches());
+        client.prepareResponse(new FetchResponse(Errors.NONE, fetchPartitionData, 0, INVALID_SESSION_ID));
+        consumerClient.poll(0);
+
+        Map>> fetchedRecords = fetcher.fetchedRecords();
+        assertEquals(3, fetchedRecords.get(tp1).size());
+        assertEquals(3, fetchedRecords.get(tp2).size());
+
+        Map allMetrics = metrics.metrics();
+        KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
+        KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
         assertEquals(expectedBytes, fetchSizeAverage.value(), EPSILON);
-        assertEquals(3, recordsCountAverage.value(), EPSILON);
+        assertEquals(6, recordsCountAverage.value(), EPSILON);
     }
 
     @Test

From 5df535e8a349771942050f1e3fd58851f413fa3a Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Sun, 25 Feb 2018 12:26:18 -0800
Subject: [PATCH 0099/1847] MINOR: fixes lgtm.com warnings (#4582)

fixes lgmt.com warnings
cleanup PrintForeachAction and Printed

Author: Matthias J. Sax 

Reviewers: Sebastian Bauersfeld , Damian Guy , Bill Bejeck , Guozhang Wang 
---
 .../apache/kafka/common/cache/LRUCache.java   |  2 +-
 .../scala/kafka/tools/StreamsResetter.java    | 30 ++++++++---------
 .../apache/kafka/streams/kstream/Printed.java | 19 ++++++-----
 .../streams/kstream/internals/KTableImpl.java | 19 ++++++-----
 .../kstream/internals/PrintForeachAction.java | 32 +++++++++----------
 .../kstream/internals/PrintedInternal.java    |  2 +-
 .../internals/InternalTopicManager.java       |  2 +-
 .../internals/StreamPartitionAssignor.java    |  4 +--
 .../processor/internals/StreamThread.java     | 12 +++----
 .../kafka/streams/kstream/PrintedTest.java    | 10 ++++--
 .../kstream/internals/KStreamPrintTest.java   | 16 ++--------
 11 files changed, 70 insertions(+), 78 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/cache/LRUCache.java b/clients/src/main/java/org/apache/kafka/common/cache/LRUCache.java
index bdc67ac2732a3..672cb65d66ab6 100644
--- a/clients/src/main/java/org/apache/kafka/common/cache/LRUCache.java
+++ b/clients/src/main/java/org/apache/kafka/common/cache/LRUCache.java
@@ -29,7 +29,7 @@ public LRUCache(final int maxSize) {
         cache = new LinkedHashMap(16, .75f, true) {
             @Override
             protected boolean removeEldestEntry(Map.Entry eldest) {
-                return size() > maxSize;
+                return this.size() > maxSize; // require this. prefix to make lgtm.com happy
             }
         };
     }
diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java
index 31c69eedc1d33..f88fce7a25b37 100644
--- a/core/src/main/scala/kafka/tools/StreamsResetter.java
+++ b/core/src/main/scala/kafka/tools/StreamsResetter.java
@@ -386,7 +386,7 @@ private void maybeReset(final String groupId,
                 shiftOffsetsBy(client, inputTopicPartitions, options.valueOf(shiftByOption));
             } else if (options.has(toDatetimeOption)) {
                 final String ts = options.valueOf(toDatetimeOption);
-                final Long timestamp = getDateTime(ts);
+                final long timestamp = getDateTime(ts);
                 resetToDatetime(client, inputTopicPartitions, timestamp);
             } else if (options.has(byDurationOption)) {
                 final String duration = options.valueOf(byDurationOption);
@@ -401,8 +401,7 @@ private void maybeReset(final String groupId,
             }
 
             for (final TopicPartition p : inputTopicPartitions) {
-                final Long position = client.position(p);
-                System.out.println("Topic: " + p.topic() + " Partition: " + p.partition() + " Offset: " + position);
+                System.out.println("Topic: " + p.topic() + " Partition: " + p.partition() + " Offset: " + client.position(p));
             }
         }
     }
@@ -416,8 +415,7 @@ public void resetOffsetsFromResetPlan(Consumer client, Set getTopicPartitionOffsetFromResetPlan(String re
     private void resetByDuration(Consumer client, Set inputTopicPartitions, Duration duration) throws DatatypeConfigurationException {
         final Date now = new Date();
         duration.negate().addTo(now);
-        final Long timestamp = now.getTime();
+        final long timestamp = now.getTime();
 
         final Map topicPartitionsAndTimes = new HashMap<>(inputTopicPartitions.size());
         for (final TopicPartition topicPartition : inputTopicPartitions) {
@@ -439,8 +437,7 @@ private void resetByDuration(Consumer client, Set topicPartitionsAndOffset = client.offsetsForTimes(topicPartitionsAndTimes);
 
         for (final TopicPartition topicPartition : inputTopicPartitions) {
-            final Long offset = topicPartitionsAndOffset.get(topicPartition).offset();
-            client.seek(topicPartition, offset);
+            client.seek(topicPartition, topicPartitionsAndOffset.get(topicPartition).offset());
         }
     }
 
@@ -453,20 +450,19 @@ private void resetToDatetime(Consumer client, Set topicPartitionsAndOffset = client.offsetsForTimes(topicPartitionsAndTimes);
 
         for (final TopicPartition topicPartition : inputTopicPartitions) {
-            final Long offset = topicPartitionsAndOffset.get(topicPartition).offset();
-            client.seek(topicPartition, offset);
+            client.seek(topicPartition, topicPartitionsAndOffset.get(topicPartition).offset());
         }
     }
 
     // visible for testing
-    public void shiftOffsetsBy(Consumer client, Set inputTopicPartitions, Long shiftBy) {
+    public void shiftOffsetsBy(Consumer client, Set inputTopicPartitions, long shiftBy) {
         final Map endOffsets = client.endOffsets(inputTopicPartitions);
         final Map beginningOffsets = client.beginningOffsets(inputTopicPartitions);
 
         final Map topicPartitionsAndOffset = new HashMap<>(inputTopicPartitions.size());
         for (final TopicPartition topicPartition : inputTopicPartitions) {
-            final Long position = client.position(topicPartition);
-            final Long offset = position + shiftBy;
+            final long position = client.position(topicPartition);
+            final long offset = position + shiftBy;
             topicPartitionsAndOffset.put(topicPartition, offset);
         }
 
@@ -497,7 +493,7 @@ public void resetOffsetsTo(Consumer client, Set
     }
 
     // visible for testing
-    public Long getDateTime(String timestamp) throws ParseException {
+    public long getDateTime(String timestamp) throws ParseException {
         final String[] timestampParts = timestamp.split("T");
         if (timestampParts.length < 2) {
             throw new ParseException("Error parsing timestamp. It does not contain a 'T' according to ISO8601 format", timestamp.length());
@@ -549,10 +545,10 @@ private Map checkOffsetRange(final Map endOffsets) {
         final Map validatedTopicPartitionsOffsets = new HashMap<>();
         for (final Map.Entry topicPartitionAndOffset : inputTopicPartitionsAndOffset.entrySet()) {
-            final Long endOffset = endOffsets.get(topicPartitionAndOffset.getKey());
-            final Long offset = topicPartitionAndOffset.getValue();
+            final long endOffset = endOffsets.get(topicPartitionAndOffset.getKey());
+            final long offset = topicPartitionAndOffset.getValue();
             if (offset < endOffset) {
-                final Long beginningOffset = beginningOffsets.get(topicPartitionAndOffset.getKey());
+                final long beginningOffset = beginningOffsets.get(topicPartitionAndOffset.getKey());
                 if (offset > beginningOffset) {
                     validatedTopicPartitionsOffsets.put(topicPartitionAndOffset.getKey(), offset);
                 } else {
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java
index 8d2c22a611583..5a1d07fd38f0b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java
@@ -19,9 +19,8 @@
 import org.apache.kafka.streams.errors.TopologyException;
 
 import java.io.FileNotFoundException;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
 import java.util.Objects;
 
 /**
@@ -32,7 +31,7 @@
  * @see KStream#print(Printed)
  */
 public class Printed {
-    protected final PrintWriter printWriter;
+    protected final OutputStream outputStream;
     protected String label;
     protected KeyValueMapper mapper = new KeyValueMapper() {
         @Override
@@ -41,8 +40,8 @@ public String apply(final K key, final V value) {
         }
     };
 
-    private Printed(final PrintWriter printWriter) {
-        this.printWriter = printWriter;
+    private Printed(final OutputStream outputStream) {
+        this.outputStream = outputStream;
     }
 
     /**
@@ -50,7 +49,7 @@ private Printed(final PrintWriter printWriter) {
      * @param printed   instance of {@link Printed} to copy
      */
     protected Printed(final Printed printed) {
-        this.printWriter = printed.printWriter;
+        this.outputStream = printed.outputStream;
         this.label = printed.label;
         this.mapper = printed.mapper;
     }
@@ -69,8 +68,8 @@ public static  Printed toFile(final String filePath) {
             throw new TopologyException("filePath can't be an empty string");
         }
         try {
-            return new Printed<>(new PrintWriter(filePath, StandardCharsets.UTF_8.name()));
-        } catch (final FileNotFoundException | UnsupportedEncodingException e) {
+            return new Printed<>(new FileOutputStream(filePath));
+        } catch (final FileNotFoundException e) {
             throw new TopologyException("Unable to write stream to file at [" + filePath + "] " + e.getMessage());
         }
     }
@@ -83,7 +82,7 @@ public static  Printed toFile(final String filePath) {
      * @return a new Printed instance
      */
     public static  Printed toSysOut() {
-        return new Printed<>((PrintWriter) null);
+        return new Printed<>(System.out);
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index a746d31289d45..11b8c516a4d25 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -39,9 +39,7 @@
 import org.apache.kafka.streams.state.StoreBuilder;
 
 import java.io.FileNotFoundException;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
+import java.io.FileOutputStream;
 import java.util.Objects;
 import java.util.Set;
 
@@ -346,7 +344,10 @@ public void print(final Serde keySerde,
                       final String label) {
         Objects.requireNonNull(label, "label can't be null");
         final String name = builder.newProcessorName(PRINTING_NAME);
-        builder.internalTopologyBuilder.addProcessor(name, new KStreamPrint<>(new PrintForeachAction(null, defaultKeyValueMapper, label)), this.name);
+        builder.internalTopologyBuilder.addProcessor(
+            name,
+            new KStreamPrint<>(new PrintForeachAction<>(System.out, defaultKeyValueMapper, label)),
+            this.name);
     }
 
     @SuppressWarnings("deprecation")
@@ -384,11 +385,13 @@ public void writeAsText(final String filePath,
         if (filePath.trim().isEmpty()) {
             throw new TopologyException("filePath can't be an empty string");
         }
-        String name = builder.newProcessorName(PRINTING_NAME);
+        final String name = builder.newProcessorName(PRINTING_NAME);
         try {
-            PrintWriter printWriter = new PrintWriter(filePath, StandardCharsets.UTF_8.name());
-            builder.internalTopologyBuilder.addProcessor(name, new KStreamPrint<>(new PrintForeachAction(printWriter, defaultKeyValueMapper, label)), this.name);
-        } catch (final FileNotFoundException | UnsupportedEncodingException e) {
+            builder.internalTopologyBuilder.addProcessor(
+                name,
+                new KStreamPrint<>(new PrintForeachAction<>(new FileOutputStream(filePath), defaultKeyValueMapper, label)),
+                this.name);
+        } catch (final FileNotFoundException e) {
             throw new TopologyException(String.format("Unable to write stream to file at [%s] %s", filePath, e.getMessage()));
         }
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java
index dcdd44fe23eca..174319fec4bd4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java
@@ -19,26 +19,30 @@
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
 
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 
 public class PrintForeachAction implements ForeachAction {
 
     private final String label;
     private final PrintWriter printWriter;
+    private final boolean closable;
     private final KeyValueMapper mapper;
+
     /**
-     * Print customized output with given writer. The PrintWriter can be null in order to
-     * distinguish between {@code System.out} and the others. If the PrintWriter is {@code PrintWriter(System.out)},
-     * then it would close {@code System.out} output stream.
-     * 
    
-     * Afterall, not to pass in {@code PrintWriter(System.out)} but {@code null} instead.
+     * Print customized output with given writer. The {@link OutputStream} can be {@link System#out} or the others.
      *
-     * @param printWriter Use {@code System.out.println} if {@code null}.
+     * @param outputStream The output stream to write to.
      * @param mapper The mapper which can allow user to customize output will be printed.
      * @param label The given name will be printed.
      */
-    public PrintForeachAction(final PrintWriter printWriter, final KeyValueMapper mapper, final String label) {
-        this.printWriter = printWriter;
+    PrintForeachAction(final OutputStream outputStream,
+                       final KeyValueMapper mapper,
+                       final String label) {
+        this.printWriter = new PrintWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8));
+        this.closable = outputStream != System.out && outputStream != System.err;
         this.mapper = mapper;
         this.label = label;
     }
@@ -46,18 +50,14 @@ public PrintForeachAction(final PrintWriter printWriter, final KeyValueMapper printed) {
      * @return the {@code ProcessorSupplier} to be used for printing
      */
     public ProcessorSupplier build(final String processorName) {
-        return new KStreamPrint<>(new PrintForeachAction<>(printWriter, mapper, label != null ? label : processorName));
+        return new KStreamPrint<>(new PrintForeachAction<>(outputStream, mapper, label != null ? label : processorName));
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
index 05d079b0c58d1..aeff946ce1efb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
@@ -221,7 +221,7 @@ private Set validateTopicPartitions(final Collection existingTopicNamesPartitions) {
         final Set topicsToBeCreated = new HashSet<>();
         for (final InternalTopicConfig topic : topicsPartitionsMap) {
-            final Integer numberOfPartitions = topic.numberOfPartitions();
+            final int numberOfPartitions = topic.numberOfPartitions();
             if (existingTopicNamesPartitions.containsKey(topic.name())) {
                 if (!existingTopicNamesPartitions.get(topic.name()).equals(numberOfPartitions)) {
                     final String errorMsg = String.format("Existing internal topic %s has invalid partitions: " +
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
index 2a08308a2fd0f..2a26272021a5d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
@@ -377,7 +377,7 @@ public Map assign(Cluster metadata, Map allRepartitionTopicPartitions = new HashMap<>();
         for (Map.Entry entry : repartitionTopicMetadata.entrySet()) {
             final String topic = entry.getKey();
-            final Integer numPartitions = entry.getValue().numPartitions;
+            final int numPartitions = entry.getValue().numPartitions;
 
             for (int partition = 0; partition < numPartitions; partition++) {
                 allRepartitionTopicPartitions.put(new TopicPartition(topic, partition),
@@ -638,7 +638,7 @@ private void prepareTopic(final Map topicPartitio
 
         for (final InternalTopicMetadata metadata : topicPartitions.values()) {
             final InternalTopicConfig topic = metadata.config;
-            final Integer numPartitions = metadata.numPartitions;
+            final int numPartitions = metadata.numPartitions;
 
             if (numPartitions == NOT_AVAILABLE) {
                 continue;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 5e25d02973aef..61a22be5a1564 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -894,15 +894,13 @@ private void addToResetList(final TopicPartition partition, final Set records) {
-        if (records != null && !records.isEmpty()) {
-            int numAddedRecords = 0;
+        int numAddedRecords = 0;
 
-            for (final TopicPartition partition : records.partitions()) {
-                final StreamTask task = taskManager.activeTask(partition);
-                numAddedRecords += task.addRecords(partition, records.records(partition));
-            }
-            streamsMetrics.skippedRecordsSensor.record(records.count() - numAddedRecords, timerStartedMs);
+        for (final TopicPartition partition : records.partitions()) {
+            final StreamTask task = taskManager.activeTask(partition);
+            numAddedRecords += task.addRecords(partition, records.records(partition));
         }
+        streamsMetrics.skippedRecordsSensor.record(records.count() - numAddedRecords, timerStartedMs);
     }
 
     /**
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/PrintedTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/PrintedTest.java
index adec9ff9f9e47..a50fce91eeff6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/PrintedTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/PrintedTest.java
@@ -41,11 +41,12 @@ public class PrintedTest {
 
     private final PrintStream originalSysOut = System.out;
     private final ByteArrayOutputStream sysOut = new ByteArrayOutputStream();
-    private final Printed sysOutPrinter = Printed.toSysOut();
+    private Printed sysOutPrinter;
 
     @Before
     public void before() {
         System.setOut(new PrintStream(sysOut));
+        sysOutPrinter = Printed.toSysOut();
     }
 
     @After
@@ -72,7 +73,10 @@ public void shouldCreateProcessorThatPrintsToFile() throws IOException {
     @Test
     public void shouldCreateProcessorThatPrintsToStdOut() throws UnsupportedEncodingException {
         final ProcessorSupplier supplier = new PrintedInternal<>(sysOutPrinter).build("processor");
-        supplier.get().process("good", 2);
+        final Processor processor = supplier.get();
+
+        processor.process("good", 2);
+        processor.close();
         assertThat(sysOut.toString(StandardCharsets.UTF_8.name()), equalTo("[processor]: good, 2\n"));
     }
 
@@ -83,6 +87,7 @@ public void shouldPrintWithLabel() throws UnsupportedEncodingException {
                 .get();
 
         processor.process("hello", 3);
+        processor.close();
         assertThat(sysOut.toString(StandardCharsets.UTF_8.name()), equalTo("[label]: hello, 3\n"));
     }
 
@@ -97,6 +102,7 @@ public String apply(final String key, final Integer value) {
                 })).build("processor")
                 .get();
         processor.process("hello", 1);
+        processor.close();
         assertThat(sysOut.toString(StandardCharsets.UTF_8.name()), equalTo("[processor]: hello -> 1\n"));
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPrintTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPrintTest.java
index e1a014d413308..3ba88e7834e99 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPrintTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPrintTest.java
@@ -16,22 +16,16 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.common.serialization.Serde;
-import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
-
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
 import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
@@ -39,9 +33,6 @@
 
 public class KStreamPrintTest {
 
-    private final Serde intSerd = Serdes.Integer();
-    private final Serde stringSerd = Serdes.String();
-    private PrintWriter printWriter;
     private ByteArrayOutputStream byteOutStream;
 
     private KeyValueMapper mapper;
@@ -49,9 +40,8 @@ public class KStreamPrintTest {
     private Processor printProcessor;
 
     @Before
-    public void setUp() {
+    public void setUp() throws Exception {
         byteOutStream = new ByteArrayOutputStream();
-        printWriter = new PrintWriter(new OutputStreamWriter(byteOutStream, StandardCharsets.UTF_8));
 
         mapper = new KeyValueMapper() {
             @Override
@@ -60,7 +50,7 @@ public String apply(final Integer key, final String value) {
             }
         };
 
-        kStreamPrint = new KStreamPrint<>(new PrintForeachAction<>(printWriter, mapper, "test-stream"));
+        kStreamPrint = new KStreamPrint<>(new PrintForeachAction<>(byteOutStream, mapper, "test-stream"));
 
         printProcessor = kStreamPrint.get();
         ProcessorContext processorContext = EasyMock.createNiceMock(ProcessorContext.class);
@@ -98,7 +88,7 @@ private  void doTest(final List> inputRecords, final String
         for (KeyValue record: inputRecords) {
             printProcessor.process(record.key, record.value);
         }
-        printWriter.flush();
+        printProcessor.close();
         assertFlushData(expectedResult, byteOutStream);
     }
 }

From 3600316a1490e81e4efafc15dd0ac31025b76e1c Mon Sep 17 00:00:00 2001
From: Gilles Degols 
Date: Mon, 26 Feb 2018 18:00:39 +0100
Subject: [PATCH 0100/1847] KAFKA-6057: Users forget `--execute` in the offset
 reset tool (#4069)

Add a small warning note when the user does not use the --execute flag.
---
 .../scala/kafka/admin/ConsumerGroupCommand.scala   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
index 68186315a4220..cb9fbe3313eda 100755
--- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
+++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala
@@ -1033,16 +1033,16 @@ object ConsumerGroupCommand extends Logging {
         CommandLineUtils.printUsageAndDie(parser, s"Option $deleteOpt either takes $groupOpt, $topicOpt, or both")
 
       if (options.has(resetOffsetsOpt)) {
-        if (!options.has(dryRunOpt) && !options.has(executeOpt)) {
-          Console.err.println("WARN: In a future major release, the default behavior of this command will be to " +
-            "prompt the user before executing the reset rather than doing a dry run. You should add the --dry-run " +
-            "option explicitly if you are scripting this command and want to keep the current default behavior " +
-            "without prompting.")
-        }
-
         if (options.has(dryRunOpt) && options.has(executeOpt))
           CommandLineUtils.printUsageAndDie(parser, s"Option $resetOffsetsOpt only accepts one of $executeOpt and $dryRunOpt")
 
+        if (!options.has(dryRunOpt) && !options.has(executeOpt)) {
+          Console.err.println("WARN: No action will be performed as the --execute option is missing." +
+            "In a future major release, the default behavior of this command will be to prompt the user before " +
+            "executing the reset rather than doing a dry run. You should add the --dry-run option explicitly " +
+            "if you are scripting this command and want to keep the current default behavior without prompting.")
+        }
+
         CommandLineUtils.checkRequiredArgs(parser, options, groupOpt)
         CommandLineUtils.checkInvalidArgs(parser, options, resetToOffsetOpt, allResetOffsetScenarioOpts - resetToOffsetOpt)
         CommandLineUtils.checkInvalidArgs(parser, options, resetToDatetimeOpt, allResetOffsetScenarioOpts - resetToDatetimeOpt)

From 2591a469305532ba79497dd435f58a9051b689db Mon Sep 17 00:00:00 2001
From: huxi 
Date: Tue, 27 Feb 2018 03:26:34 +0800
Subject: [PATCH 0101/1847] KAFKA-5327; Console Consumer should not commit
 messages not printed (#4546)

Ensure that the consumer's offsets are reset prior to closing so that any buffered messages which haven't been printed are not committed.
---
 .../scala/kafka/consumer/BaseConsumer.scala   | 19 +++++--
 .../scala/kafka/tools/ConsoleConsumer.scala   | 14 +++--
 .../kafka/tools/ConsoleConsumerTest.scala     | 53 +++++++++++++++++--
 3 files changed, 71 insertions(+), 15 deletions(-)

diff --git a/core/src/main/scala/kafka/consumer/BaseConsumer.scala b/core/src/main/scala/kafka/consumer/BaseConsumer.scala
index 04ac2d9f1a570..9e49fe4dc8b39 100644
--- a/core/src/main/scala/kafka/consumer/BaseConsumer.scala
+++ b/core/src/main/scala/kafka/consumer/BaseConsumer.scala
@@ -23,6 +23,7 @@ import java.util.regex.Pattern
 import kafka.api.OffsetRequest
 import kafka.common.StreamEndException
 import kafka.message.Message
+import org.apache.kafka.clients.consumer.Consumer
 import org.apache.kafka.common.record.TimestampType
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.header.Headers
@@ -55,10 +56,8 @@ case class BaseConsumerRecord(topic: String,
 
 @deprecated("This class has been deprecated and will be removed in a future release. " +
             "Please use org.apache.kafka.clients.consumer.KafkaConsumer instead.", "0.11.0.0")
-class NewShinyConsumer(topic: Option[String], partitionId: Option[Int], offset: Option[Long], whitelist: Option[String], consumerProps: Properties, val timeoutMs: Long = Long.MaxValue) extends BaseConsumer {
-  import org.apache.kafka.clients.consumer.KafkaConsumer
-
-  val consumer = new KafkaConsumer[Array[Byte], Array[Byte]](consumerProps)
+class NewShinyConsumer(topic: Option[String], partitionId: Option[Int], offset: Option[Long], whitelist: Option[String],
+                       consumer: Consumer[Array[Byte], Array[Byte]], val timeoutMs: Long = Long.MaxValue) extends BaseConsumer {
   consumerInit()
   var recordIter = consumer.poll(0).iterator
 
@@ -91,6 +90,17 @@ class NewShinyConsumer(topic: Option[String], partitionId: Option[Int], offset:
     }
   }
 
+  def resetUnconsumedOffsets() {
+    val smallestUnconsumedOffsets = collection.mutable.Map[TopicPartition, Long]()
+    while (recordIter.hasNext) {
+      val record = recordIter.next()
+      val tp = new TopicPartition(record.topic, record.partition)
+      // avoid auto-committing offsets which haven't been consumed
+      smallestUnconsumedOffsets.getOrElseUpdate(tp, record.offset)
+    }
+    smallestUnconsumedOffsets.foreach { case (tp, offset) => consumer.seek(tp, offset) }
+  }
+
   override def receive(): BaseConsumerRecord = {
     if (!recordIter.hasNext) {
       recordIter = consumer.poll(timeoutMs).iterator
@@ -114,6 +124,7 @@ class NewShinyConsumer(topic: Option[String], partitionId: Option[Int], offset:
   }
 
   override def cleanup() {
+    resetUnconsumedOffsets()
     this.consumer.close()
   }
 
diff --git a/core/src/main/scala/kafka/tools/ConsoleConsumer.scala b/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
index 7d2d371015783..24fa583f1bbbc 100755
--- a/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
+++ b/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
@@ -31,10 +31,10 @@ import kafka.message._
 import kafka.metrics.KafkaMetricsReporter
 import kafka.utils._
 import kafka.utils.Implicits._
-import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
+import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, KafkaConsumer}
 import org.apache.kafka.common.errors.{AuthenticationException, WakeupException}
 import org.apache.kafka.common.record.TimestampType
-import org.apache.kafka.common.serialization.Deserializer
+import org.apache.kafka.common.serialization.{ByteArrayDeserializer, Deserializer}
 import org.apache.kafka.common.utils.Utils
 
 import scala.collection.JavaConverters._
@@ -72,10 +72,11 @@ object ConsoleConsumer extends Logging {
         new OldConsumer(conf.filterSpec, props)
       } else {
         val timeoutMs = if (conf.timeoutMs >= 0) conf.timeoutMs else Long.MaxValue
+        val consumer = new KafkaConsumer(getNewConsumerProps(conf), new ByteArrayDeserializer, new ByteArrayDeserializer)
         if (conf.partitionArg.isDefined)
-          new NewShinyConsumer(Option(conf.topicArg), conf.partitionArg, Option(conf.offsetArg), None, getNewConsumerProps(conf), timeoutMs)
+          new NewShinyConsumer(Option(conf.topicArg), conf.partitionArg, Option(conf.offsetArg), None, consumer, timeoutMs)
         else
-          new NewShinyConsumer(Option(conf.topicArg), None, None, Option(conf.whitelistArg), getNewConsumerProps(conf), timeoutMs)
+          new NewShinyConsumer(Option(conf.topicArg), None, None, Option(conf.whitelistArg), consumer, timeoutMs)
       }
 
     addShutdownHook(consumer, conf)
@@ -202,15 +203,12 @@ object ConsoleConsumer extends Logging {
     }
   }
 
-  def getNewConsumerProps(config: ConsumerConfig): Properties = {
+  private[tools] def getNewConsumerProps(config: ConsumerConfig): Properties = {
     val props = new Properties
-
     props ++= config.consumerProps
     props ++= config.extraConsumerProps
     setAutoOffsetResetValue(config, props)
     props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.bootstrapServer)
-    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer")
-    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer")
     props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, config.isolationLevel)
     props
   }
diff --git a/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala b/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala
index 364f6b3b93e3b..9ae8b966ac045 100644
--- a/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala
+++ b/core/src/test/scala/unit/kafka/tools/ConsoleConsumerTest.scala
@@ -17,18 +17,65 @@
 
 package kafka.tools
 
-import java.io.{PrintStream, FileOutputStream}
+import java.io.{FileOutputStream, PrintStream}
 
 import kafka.common.MessageFormatter
-import kafka.consumer.{BaseConsumer, BaseConsumerRecord}
+import kafka.consumer.{BaseConsumer, BaseConsumerRecord, NewShinyConsumer}
 import kafka.utils.{Exit, TestUtils}
+import org.apache.kafka.clients.consumer.{ConsumerRecord, MockConsumer, OffsetResetStrategy}
+import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.clients.consumer.ConsumerConfig
 import org.easymock.EasyMock
 import org.junit.Assert._
-import org.junit.Test
+import org.junit.{Before, Test}
+
+import scala.collection.JavaConverters._
 
 class ConsoleConsumerTest {
 
+  @Before
+  def setup(): Unit = {
+    ConsoleConsumer.messageCount = 0
+  }
+
+  @Test
+  def shouldResetUnConsumedOffsetsBeforeExitForNewConsumer() {
+    val topic = "test"
+    val maxMessages: Int = 123
+    val totalMessages: Int = 700
+    val startOffset: java.lang.Long = 0L
+
+    val mockConsumer = new MockConsumer[Array[Byte], Array[Byte]](OffsetResetStrategy.EARLIEST)
+    val tp1 = new TopicPartition(topic, 0)
+    val tp2 = new TopicPartition(topic, 1)
+
+    val consumer = new NewShinyConsumer(Some(topic), None, None, None, mockConsumer)
+
+    mockConsumer.rebalance(List(tp1, tp2).asJava)
+    mockConsumer.updateBeginningOffsets(Map(tp1 -> startOffset, tp2 -> startOffset).asJava)
+
+    0 until totalMessages foreach { i =>
+      // add all records, each partition should have half of `totalMessages`
+      mockConsumer.addRecord(new ConsumerRecord[Array[Byte], Array[Byte]](topic, i % 2, i / 2, "key".getBytes, "value".getBytes))
+    }
+
+    // Mocks
+    val formatter = EasyMock.createNiceMock(classOf[MessageFormatter])
+
+    // Expectations
+    EasyMock.expect(formatter.writeTo(EasyMock.anyObject(), EasyMock.anyObject())).times(maxMessages)
+    EasyMock.replay(formatter)
+
+    // Test
+    ConsoleConsumer.process(maxMessages, formatter, consumer, System.out, skipMessageOnError = false)
+    assertEquals(totalMessages, mockConsumer.position(tp1) + mockConsumer.position(tp2))
+
+    consumer.resetUnconsumedOffsets()
+    assertEquals(maxMessages, mockConsumer.position(tp1) + mockConsumer.position(tp2))
+
+    EasyMock.verify(formatter)
+  }
+
   @Test
   def shouldLimitReadsToMaxMessageLimit() {
     //Mocks

From 7c5d0c459f1d4a8796e3b5af925f40fd4f8a69fa Mon Sep 17 00:00:00 2001
From: Blake Miller 
Date: Mon, 26 Feb 2018 20:25:53 +0000
Subject: [PATCH 0102/1847] MINOR:Fix typo in the impl source (#4587)

The static method KStreamImpl.createReparitionedSource() is missing a t.

This PR globally fixes the typo and keeps the code indentation consistent.
---
 .../internals/GroupedStreamAggregateBuilder.java     |  2 +-
 .../kstream/internals/KGroupedStreamImpl.java        |  2 +-
 .../kafka/streams/kstream/internals/KStreamImpl.java | 12 ++++++------
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java
index e4429cc5e881d..24ed8a0c49f38 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java
@@ -86,6 +86,6 @@ private String repartitionIfRequired(final String queryableStoreName) {
         if (!repartitionRequired) {
             return this.name;
         }
-        return KStreamImpl.createReparitionedSource(builder, keySerde, valueSerde, queryableStoreName, name);
+        return KStreamImpl.createRepartitionedSource(builder, keySerde, valueSerde, queryableStoreName, name);
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
index 45ae7da7e0f44..82e2823c3ed02 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
@@ -519,6 +519,6 @@ private String repartitionIfRequired(final String queryableStoreName) {
         if (!repartitionRequired) {
             return this.name;
         }
-        return KStreamImpl.createReparitionedSource(builder, keySerde, valSerde, queryableStoreName, name);
+        return KStreamImpl.createRepartitionedSource(builder, keySerde, valSerde, queryableStoreName, name);
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
index 8aaab49a7d872..141bbbb6c55c7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java
@@ -653,16 +653,16 @@ private  KStream doJoin(final KStream other,
      */
     private KStreamImpl repartitionForJoin(final Serde keySerde,
                                                  final Serde valSerde) {
-        String repartitionedSourceName = createReparitionedSource(builder, keySerde, valSerde, null, name);
+        String repartitionedSourceName = createRepartitionedSource(builder, keySerde, valSerde, null, name);
         return new KStreamImpl<>(builder, repartitionedSourceName, Collections
             .singleton(repartitionedSourceName), false);
     }
 
-    static  String createReparitionedSource(final InternalStreamsBuilder builder,
-                                                    final Serde keySerde,
-                                                    final Serde valSerde,
-                                                    final String topicNamePrefix,
-                                                    final String name) {
+    static  String createRepartitionedSource(final InternalStreamsBuilder builder,
+                                                     final Serde keySerde,
+                                                     final Serde valSerde,
+                                                     final String topicNamePrefix,
+                                                     final String name) {
         Serializer keySerializer = keySerde != null ? keySerde.serializer() : null;
         Serializer valSerializer = valSerde != null ? valSerde.serializer() : null;
         Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null;

From f26fbb9adcd66a31740da5c99f14a108bbc24304 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 26 Feb 2018 14:39:47 -0800
Subject: [PATCH 0103/1847] MINOR: Rename stream partition assignor to streams
 partition assignor (#4621)

This is a straight-forward change that make the name of the partition assignor to be aligned with Streams.

Reviewers: Matthias J. Sax 
---
 checkstyle/suppressions.xml                   | 10 +++---
 .../apache/kafka/streams/StreamsConfig.java   |  4 +--
 .../processor/DefaultPartitionGrouper.java    |  4 +--
 .../streams/processor/TopologyBuilder.java    |  2 +-
 ...nor.java => StreamsPartitionAssignor.java} |  2 +-
 .../apache/kafka/streams/state/HostInfo.java  |  4 +--
 .../kafka/streams/StreamsConfigTest.java      |  4 +--
 .../processor/TopologyBuilderTest.java        |  6 ++--
 .../CopartitionedTopicsValidatorTest.java     | 36 +++++++++----------
 ...java => StreamsPartitionAssignorTest.java} |  6 ++--
 10 files changed, 39 insertions(+), 39 deletions(-)
 rename streams/src/main/java/org/apache/kafka/streams/processor/internals/{StreamPartitionAssignor.java => StreamsPartitionAssignor.java} (99%)
 rename streams/src/test/java/org/apache/kafka/streams/processor/internals/{StreamPartitionAssignorTest.java => StreamsPartitionAssignorTest.java} (99%)

diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index f23805e665f93..45ee4e60a7c07 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -134,7 +134,7 @@
               files="(TopologyBuilder|KafkaStreams|KStreamImpl|KTableImpl|StreamThread|StreamTask).java"/>
 
     
+              files="StreamsPartitionAssignor.java"/>
 
     
@@ -142,22 +142,22 @@
               files="RocksDBWindowStoreSupplier.java"/>
 
     
+              files="(TopologyBuilder|KStreamImpl|StreamsPartitionAssignor|KafkaStreams|KTableImpl).java"/>
 
     
     
+              files="StreamsPartitionAssignor.java"/>
     
 
     
+              files="StreamsPartitionAssignor.java"/>
 
     
     
+              files="StreamsPartitionAssignor.java"/>
     
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index ec0a1a8385b58..6b3626101bdb6 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -39,7 +39,7 @@
 import org.apache.kafka.streams.processor.DefaultPartitionGrouper;
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
 import org.apache.kafka.streams.processor.TimestampExtractor;
-import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor;
+import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -782,7 +782,7 @@ public Map getConsumerConfigs(final String groupId,
         consumerProps.put(REPLICATION_FACTOR_CONFIG, getInt(REPLICATION_FACTOR_CONFIG));
         consumerProps.put(APPLICATION_SERVER_CONFIG, getString(APPLICATION_SERVER_CONFIG));
         consumerProps.put(NUM_STANDBY_REPLICAS_CONFIG, getInt(NUM_STANDBY_REPLICAS_CONFIG));
-        consumerProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, StreamPartitionAssignor.class.getName());
+        consumerProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, StreamsPartitionAssignor.class.getName());
         consumerProps.put(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG, getLong(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG));
 
         // add admin retries configs for creating topics
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java b/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java
index 19e480908d2be..c86171c3ab06a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/DefaultPartitionGrouper.java
@@ -20,7 +20,7 @@
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.streams.errors.StreamsException;
-import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor;
+import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -83,7 +83,7 @@ protected int maxNumPartitions(Cluster metadata, Set topics) {
 
             if (partitions.isEmpty()) {
                 log.info("Skipping assigning topic {} to tasks since its metadata is not available yet", topic);
-                return StreamPartitionAssignor.NOT_AVAILABLE;
+                return StreamsPartitionAssignor.NOT_AVAILABLE;
             } else {
                 int numPartitions = partitions.size();
                 if (numPartitions > maxNumPartitions)
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
index 6f34e25a7b72d..dab7bd793d1d4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
@@ -29,7 +29,7 @@
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.SinkNode;
 import org.apache.kafka.streams.processor.internals.SourceNode;
-import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor.SubscriptionUpdates;
+import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor.SubscriptionUpdates;
 import org.apache.kafka.streams.state.KeyValueStore;
 
 import java.util.Collection;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
similarity index 99%
rename from streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
rename to streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 2a26272021a5d..9aa0e94c8c1ba 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -54,7 +54,7 @@
 import static org.apache.kafka.common.utils.Utils.getHost;
 import static org.apache.kafka.common.utils.Utils.getPort;
 
-public class StreamPartitionAssignor implements PartitionAssignor, Configurable {
+public class StreamsPartitionAssignor implements PartitionAssignor, Configurable {
 
     private final static int UNKNOWN = -1;
     public final static int NOT_AVAILABLE = -2;
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java b/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java
index c1b102107d15b..58cdba6b8fb09 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java
@@ -19,7 +19,7 @@
 import org.apache.kafka.common.serialization.Serializer;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.processor.StreamPartitioner;
-import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor;
+import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
 
 /**
  * Represents a user defined endpoint in a {@link org.apache.kafka.streams.KafkaStreams} application.
@@ -30,7 +30,7 @@
  *  {@link KafkaStreams#metadataForKey(String, Object, Serializer)}
  *
  *  The HostInfo is constructed during Partition Assignment
- *  see {@link StreamPartitionAssignor}
+ *  see {@link StreamsPartitionAssignor}
  *  It is extracted from the config {@link org.apache.kafka.streams.StreamsConfig#APPLICATION_SERVER_CONFIG}
  *
  *  If developers wish to expose an endpoint in their KafkaStreams applications they should provide the above
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index cc072d5508d99..0309659042d0d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -30,7 +30,7 @@
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
 import org.apache.kafka.streams.processor.TimestampExtractor;
-import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor;
+import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
 import org.hamcrest.CoreMatchers;
 import org.junit.Before;
 import org.junit.Test;
@@ -119,7 +119,7 @@ public void consumerConfigMustContainStreamPartitionAssignorConfig() {
 
         assertEquals(42, returnedProps.get(StreamsConfig.REPLICATION_FACTOR_CONFIG));
         assertEquals(1, returnedProps.get(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG));
-        assertEquals(StreamPartitionAssignor.class.getName(), returnedProps.get(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG));
+        assertEquals(StreamsPartitionAssignor.class.getName(), returnedProps.get(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG));
         assertEquals(7L, returnedProps.get(StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG));
         assertEquals("dummy:host", returnedProps.get(StreamsConfig.APPLICATION_SERVER_CONFIG));
         assertEquals(null, returnedProps.get(StreamsConfig.RETRIES_CONFIG));
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
index ac3cd49923bde..7a815944ecd7c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
@@ -27,7 +27,7 @@
 import org.apache.kafka.streams.processor.internals.ProcessorNode;
 import org.apache.kafka.streams.processor.internals.ProcessorStateManager;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
-import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor;
+import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
 import org.apache.kafka.streams.processor.internals.UnwindowedChangelogTopicConfig;
 import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier;
@@ -661,7 +661,7 @@ public void shouldSetCorrectSourceNodesWithRegexUpdatedTopics() throws Exception
         builder.addSource("source-2", Pattern.compile("topic-[A-C]"));
         builder.addSource("source-3", Pattern.compile("topic-\\d"));
 
-        StreamPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamPartitionAssignor.SubscriptionUpdates();
+        StreamsPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamsPartitionAssignor.SubscriptionUpdates();
         Field updatedTopicsField  = subscriptionUpdates.getClass().getDeclaredField("updatedTopicSubscriptions");
         updatedTopicsField.setAccessible(true);
 
@@ -741,7 +741,7 @@ public void shouldConnectRegexMatchedTopicsToStateStore() throws Exception {
                 .addProcessor("my-processor", new MockProcessorSupplier(), "ingest")
                 .addStateStore(new MockStateStoreSupplier("testStateStore", false), "my-processor");
 
-        final StreamPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamPartitionAssignor.SubscriptionUpdates();
+        final StreamsPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamsPartitionAssignor.SubscriptionUpdates();
         final Field updatedTopicsField  = subscriptionUpdates.getClass().getDeclaredField("updatedTopicSubscriptions");
         updatedTopicsField.setAccessible(true);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
index 19277e902ae90..bbc59fa6fa98a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsValidatorTest.java
@@ -33,8 +33,8 @@
 
 public class CopartitionedTopicsValidatorTest {
 
-    private final StreamPartitionAssignor.CopartitionedTopicsValidator validator
-            = new StreamPartitionAssignor.CopartitionedTopicsValidator("thread");
+    private final StreamsPartitionAssignor.CopartitionedTopicsValidator validator
+            = new StreamsPartitionAssignor.CopartitionedTopicsValidator("thread");
     private final Map partitions = new HashMap<>();
     private final Cluster cluster = Cluster.empty();
 
@@ -49,7 +49,7 @@ public void before() {
     @Test(expected = TopologyBuilderException.class)
     public void shouldThrowTopologyBuilderExceptionIfNoPartitionsFoundForCoPartitionedTopic() {
         validator.validate(Collections.singleton("topic"),
-                           Collections.emptyMap(),
+                           Collections.emptyMap(),
                            cluster);
     }
 
@@ -57,14 +57,14 @@ public void shouldThrowTopologyBuilderExceptionIfNoPartitionsFoundForCoPartition
     public void shouldThrowTopologyBuilderExceptionIfPartitionCountsForCoPartitionedTopicsDontMatch() {
         partitions.remove(new TopicPartition("second", 0));
         validator.validate(Utils.mkSet("first", "second"),
-                           Collections.emptyMap(),
+                           Collections.emptyMap(),
                            cluster.withPartitions(partitions));
     }
 
 
     @Test
     public void shouldEnforceCopartitioningOnRepartitionTopics() {
-        final StreamPartitionAssignor.InternalTopicMetadata metadata = createTopicMetadata("repartitioned", 10);
+        final StreamsPartitionAssignor.InternalTopicMetadata metadata = createTopicMetadata("repartitioned", 10);
 
         validator.validate(Utils.mkSet("first", "second", metadata.config.name()),
                            Collections.singletonMap(metadata.config.name(),
@@ -77,10 +77,10 @@ public void shouldEnforceCopartitioningOnRepartitionTopics() {
 
     @Test
     public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartitionTopics() {
-        final StreamPartitionAssignor.InternalTopicMetadata one = createTopicMetadata("one", 1);
-        final StreamPartitionAssignor.InternalTopicMetadata two = createTopicMetadata("two", 15);
-        final StreamPartitionAssignor.InternalTopicMetadata three = createTopicMetadata("three", 5);
-        final Map repartitionTopicConfig = new HashMap<>();
+        final StreamsPartitionAssignor.InternalTopicMetadata one = createTopicMetadata("one", 1);
+        final StreamsPartitionAssignor.InternalTopicMetadata two = createTopicMetadata("two", 15);
+        final StreamsPartitionAssignor.InternalTopicMetadata three = createTopicMetadata("three", 5);
+        final Map repartitionTopicConfig = new HashMap<>();
 
         repartitionTopicConfig.put(one.config.name(), one);
         repartitionTopicConfig.put(two.config.name(), two);
@@ -100,9 +100,9 @@ public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartition
 
     @Test
     public void shouldSetRepartitionTopicsPartitionCountToNotAvailableIfAnyNotAvaliable() {
-        final StreamPartitionAssignor.InternalTopicMetadata one = createTopicMetadata("one", 1);
-        final StreamPartitionAssignor.InternalTopicMetadata two = createTopicMetadata("two", StreamPartitionAssignor.NOT_AVAILABLE);
-        final Map repartitionTopicConfig = new HashMap<>();
+        final StreamsPartitionAssignor.InternalTopicMetadata one = createTopicMetadata("one", 1);
+        final StreamsPartitionAssignor.InternalTopicMetadata two = createTopicMetadata("two", StreamsPartitionAssignor.NOT_AVAILABLE);
+        final Map repartitionTopicConfig = new HashMap<>();
 
         repartitionTopicConfig.put(one.config.name(), one);
         repartitionTopicConfig.put(two.config.name(), two);
@@ -114,18 +114,18 @@ public void shouldSetRepartitionTopicsPartitionCountToNotAvailableIfAnyNotAvalia
                            repartitionTopicConfig,
                            cluster.withPartitions(partitions));
 
-        assertThat(one.numPartitions, equalTo(StreamPartitionAssignor.NOT_AVAILABLE));
-        assertThat(two.numPartitions, equalTo(StreamPartitionAssignor.NOT_AVAILABLE));
+        assertThat(one.numPartitions, equalTo(StreamsPartitionAssignor.NOT_AVAILABLE));
+        assertThat(two.numPartitions, equalTo(StreamsPartitionAssignor.NOT_AVAILABLE));
 
     }
 
-    private StreamPartitionAssignor.InternalTopicMetadata createTopicMetadata(final String repartitionTopic,
-                                                                              final int partitions) {
+    private StreamsPartitionAssignor.InternalTopicMetadata createTopicMetadata(final String repartitionTopic,
+                                                                               final int partitions) {
         final InternalTopicConfig repartitionTopicConfig
                 = new RepartitionTopicConfig(repartitionTopic, Collections.emptyMap());
 
-        final StreamPartitionAssignor.InternalTopicMetadata metadata
-                = new StreamPartitionAssignor.InternalTopicMetadata(repartitionTopicConfig);
+        final StreamsPartitionAssignor.InternalTopicMetadata metadata
+                = new StreamsPartitionAssignor.InternalTopicMetadata(repartitionTopicConfig);
         metadata.numPartitions = partitions;
         return metadata;
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
similarity index 99%
rename from streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java
rename to streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index 02ab803735aa7..bf3f1d1ac5ee3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -65,7 +65,7 @@
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertThat;
 
-public class StreamPartitionAssignorTest {
+public class StreamsPartitionAssignorTest {
 
     private final TopicPartition t1p0 = new TopicPartition("topic1", 0);
     private final TopicPartition t1p1 = new TopicPartition("topic1", 1);
@@ -105,7 +105,7 @@ public class StreamPartitionAssignorTest {
     private final TaskId task1 = new TaskId(0, 1);
     private final TaskId task2 = new TaskId(0, 2);
     private final TaskId task3 = new TaskId(0, 3);
-    private final StreamPartitionAssignor partitionAssignor = new StreamPartitionAssignor();
+    private final StreamsPartitionAssignor partitionAssignor = new StreamsPartitionAssignor();
     private final MockClientSupplier mockClientSupplier = new MockClientSupplier();
     private final InternalTopologyBuilder builder = new InternalTopologyBuilder();
     private final StreamsConfig streamsConfig = new StreamsConfig(configProps());
@@ -762,7 +762,7 @@ public KeyValue apply(final Object key, final Object value) {
             .count();
 
         // joining the stream and the table
-        // this triggers the enforceCopartitioning() routine in the StreamPartitionAssignor,
+        // this triggers the enforceCopartitioning() routine in the StreamsPartitionAssignor,
         // forcing the stream.map to get repartitioned to a topic with four partitions.
         stream1.join(
             table1,

From 97ad549d56d3432b0c70e72d402ebec1d07f8273 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 27 Feb 2018 00:29:25 -0800
Subject: [PATCH 0104/1847] KAFKA-6534: Enforce a rebalance in the next poll
 call when encounter task migration (#4544)

The fix is in two folds:

For tasks that's closed in closeZombieTask, their corresponding partitions are still in runningByPartition so those closed tasks may still be returned in activeTasks and standbyTasks. Adding guards on the returned tasks and if they are closed notify the thread to trigger rebalance immediately.

When triggering a rebalance, un-subscribe and re-subscribe immediately to make sure we are not dependent on the background heartbeat thread timing.

Some minor changes on log4j. More specifically, I moved the log entry of closeZombieTask to its callers with more context information and the action going to take.

I can re-produce the issue with EosIntegrationTest may hand-code the heartbeat thread to GC, and confirmed this patch fixed the issue. Unfortunately this test cannot be added to AK since currently we do not have ways to manipulate the heartbeat thread in unit tests.

Reviewers: Jason Gustafson , Bill Bejeck , Matthias J. Sax 
---
 .../internals/AbstractCoordinator.java        |  8 ++---
 .../processor/internals/AbstractTask.java     |  4 +++
 .../internals/AssignedStreamsTasks.java       |  4 +++
 .../processor/internals/AssignedTasks.java    | 14 +++++---
 .../processor/internals/StandbyTask.java      |  2 ++
 .../processor/internals/StreamTask.java       |  2 ++
 .../processor/internals/StreamThread.java     | 34 ++++++++++++++++++-
 .../processor/internals/TaskManager.java      |  1 -
 .../integration/EosIntegrationTest.java       | 16 +++++----
 9 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 6884ff0dff754..b39f52c055219 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -767,20 +767,20 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
                 future.complete(null);
             } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                     || error == Errors.NOT_COORDINATOR) {
-                log.debug("Attempt to heartbeat since coordinator {} is either not started or not valid.",
+                log.info("Attempt to heartbeat failed since coordinator {} is either not started or not valid.",
                         coordinator());
                 coordinatorDead();
                 future.raise(error);
             } else if (error == Errors.REBALANCE_IN_PROGRESS) {
-                log.debug("Attempt to heartbeat failed since group is rebalancing");
+                log.info("Attempt to heartbeat failed since group is rebalancing");
                 requestRejoin();
                 future.raise(Errors.REBALANCE_IN_PROGRESS);
             } else if (error == Errors.ILLEGAL_GENERATION) {
-                log.debug("Attempt to heartbeat failed since generation {} is not current", generation.generationId);
+                log.info("Attempt to heartbeat failed since generation {} is not current", generation.generationId);
                 resetGeneration();
                 future.raise(Errors.ILLEGAL_GENERATION);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
-                log.debug("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId);
+                log.info("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId);
                 resetGeneration();
                 future.raise(Errors.UNKNOWN_MEMBER_ID);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
index d9c827fff52b4..a8f7e652da956 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AbstractTask.java
@@ -52,6 +52,7 @@ public abstract class AbstractTask implements Task {
     final Logger log;
     final LogContext logContext;
     boolean taskInitialized;
+    boolean taskClosed;
     final StateDirectory stateDirectory;
 
     InternalProcessorContext processorContext;
@@ -256,6 +257,9 @@ void closeStateManager(final boolean writeCheckpoint) throws ProcessorStateExcep
         }
     }
 
+    public boolean isClosed() {
+        return taskClosed;
+    }
 
     public boolean hasStateStores() {
         return !topology.stateStores().isEmpty();
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
index 7b05f6488e781..f98e6356a228d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedStreamsTasks.java
@@ -95,6 +95,8 @@ int process() {
                     processed++;
                 }
             } catch (final TaskMigratedException e) {
+                log.info("Failed to process stream task {} since it got migrated to another thread already. " +
+                        "Closing it as zombie before triggering a new rebalance.", task.id());
                 final RuntimeException fatalException = closeZombieTask(task);
                 if (fatalException != null) {
                     throw fatalException;
@@ -125,6 +127,8 @@ int punctuate() {
                     punctuated++;
                 }
             } catch (final TaskMigratedException e) {
+                log.info("Failed to punctuate stream task {} since it got migrated to another thread already. " +
+                        "Closing it as zombie before triggering a new rebalance.", task.id());
                 final RuntimeException fatalException = closeZombieTask(task);
                 if (fatalException != null) {
                     throw fatalException;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
index 2cd82f461ddd5..8529c9eca8816 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/AssignedTasks.java
@@ -200,6 +200,8 @@ private RuntimeException suspendTasks(final Collection tasks) {
                 suspended.put(task.id(), task);
             } catch (final TaskMigratedException closeAsZombieAndSwallow) {
                 // as we suspend a task, we are either shutting down or rebalancing, thus, we swallow and move on
+                log.info("Failed to suspend {} {} since it got migrated to another thread already. " +
+                        "Closing it as zombie and move on.", taskTypeName, task.id());
                 firstException.compareAndSet(null, closeZombieTask(task));
                 it.remove();
             } catch (final RuntimeException e) {
@@ -216,7 +218,6 @@ private RuntimeException suspendTasks(final Collection tasks) {
     }
 
     RuntimeException closeZombieTask(final T task) {
-        log.warn("{} {} got migrated to another thread already. Closing it as zombie.", taskTypeName, task.id());
         try {
             task.close(false, true);
         } catch (final RuntimeException e) {
@@ -242,11 +243,12 @@ boolean maybeResumeSuspendedTask(final TaskId taskId, final Set
                 try {
                     task.resume();
                 } catch (final TaskMigratedException e) {
+                    log.info("Failed to resume {} {} since it got migrated to another thread already. " +
+                            "Closing it as zombie before triggering a new rebalance.", taskTypeName, task.id());
                     final RuntimeException fatalException = closeZombieTask(task);
                     if (fatalException != null) {
                         throw fatalException;
                     }
-                    suspended.remove(taskId);
                     throw e;
                 }
                 transitionToRunning(task, new HashSet());
@@ -368,14 +370,14 @@ void applyToRunningTasks(final TaskAction action) {
             try {
                 action.apply(task);
             } catch (final TaskMigratedException e) {
+                log.info("Failed to commit {} {} since it got migrated to another thread already. " +
+                        "Closing it as zombie before triggering a new rebalance.", taskTypeName, task.id());
                 final RuntimeException fatalException = closeZombieTask(task);
                 if (fatalException != null) {
                     throw fatalException;
                 }
                 it.remove();
-                if (firstException == null) {
-                    firstException = e;
-                }
+                throw e;
             } catch (final RuntimeException t) {
                 log.error("Failed to {} {} {} due to the following error:",
                           action.name(),
@@ -416,6 +418,8 @@ void close(final boolean clean) {
             try {
                 task.close(clean, false);
             } catch (final TaskMigratedException e) {
+                log.info("Failed to close {} {} since it got migrated to another thread already. " +
+                        "Closing it as zombie and move on.", taskTypeName, task.id());
                 firstException.compareAndSet(null, closeZombieTask(task));
             } catch (final RuntimeException t) {
                 log.error("Failed while closing {} {} due to the following error:",
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index 39d34d799d2fe..861556cded3ec 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -144,6 +144,8 @@ public void close(final boolean clean,
         } finally {
             closeStateManager(committedSuccessfully);
         }
+
+        taskClosed = true;
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 6bca02ad9bc8e..b8777ad5521f8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -541,6 +541,8 @@ public void close(boolean clean,
         }
 
         closeSuspended(clean, isZombie, firstException);
+
+        taskClosed = true;
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 61a22be5a1564..cda04e9efc027 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -757,7 +757,12 @@ private void runLoop() {
             } catch (final TaskMigratedException ignoreAndRejoinGroup) {
                 log.warn("Detected task {} that got migrated to another thread. " +
                     "This implies that this thread missed a rebalance and dropped out of the consumer group. " +
-                    "Trying to rejoin the consumer group now. Below is the detailed description of the task:\n{}", ignoreAndRejoinGroup.migratedTask().id(), ignoreAndRejoinGroup.migratedTask().toString(">"));
+                    "Will try to rejoin the consumer group. Below is the detailed description of the task:\n{}",
+                        ignoreAndRejoinGroup.migratedTask().id(), ignoreAndRejoinGroup.migratedTask().toString(">"));
+
+                // re-subscribe to enforce a rebalance in the next poll call
+                consumer.unsubscribe();
+                consumer.subscribe(builder.sourceTopicPattern(), rebalanceListener);
             }
         }
     }
@@ -898,6 +903,13 @@ private void addRecordsToTasks(final ConsumerRecords records) {
 
         for (final TopicPartition partition : records.partitions()) {
             final StreamTask task = taskManager.activeTask(partition);
+
+            if (task.isClosed()) {
+                log.warn("Stream task {} is already closed, probably because it got unexpectly migrated to another thread already. " +
+                        "Notifying the thread to trigger a new rebalance immediately.", task.id());
+                throw new TaskMigratedException(task);
+            }
+
             numAddedRecords += task.addRecords(partition, records.records(partition));
         }
         streamsMetrics.skippedRecordsSensor.record(records.count() - numAddedRecords, timerStartedMs);
@@ -1024,6 +1036,13 @@ private void maybeUpdateStandbyTasks(final long now) {
                         List> remaining = entry.getValue();
                         if (remaining != null) {
                             final StandbyTask task = taskManager.standbyTask(partition);
+
+                            if (task.isClosed()) {
+                                log.warn("Standby task {} is already closed, probably because it got unexpectly migrated to another thread already. " +
+                                        "Notifying the thread to trigger a new rebalance immediately.", task.id());
+                                throw new TaskMigratedException(task);
+                            }
+
                             remaining = task.update(partition, remaining);
                             if (remaining != null) {
                                 remainingStandbyRecords.put(partition, remaining);
@@ -1051,6 +1070,12 @@ private void maybeUpdateStandbyTasks(final long now) {
                             throw new StreamsException(logPrefix + "Missing standby task for partition " + partition);
                         }
 
+                        if (task.isClosed()) {
+                            log.warn("Standby task {} is already closed, probably because it got unexpectly migrated to another thread already. " +
+                                    "Notifying the thread to trigger a new rebalance immediately.", task.id());
+                            throw new TaskMigratedException(task);
+                        }
+
                         final List> remaining = task.update(partition, records.records(partition));
                         if (remaining != null) {
                             restoreConsumer.pause(singleton(partition));
@@ -1063,6 +1088,13 @@ private void maybeUpdateStandbyTasks(final long now) {
                 final Set partitions = recoverableException.partitions();
                 for (final TopicPartition partition : partitions) {
                     final StandbyTask task = taskManager.standbyTask(partition);
+
+                    if (task.isClosed()) {
+                        log.warn("Standby task {} is already closed, probably because it got unexpectly migrated to another thread already. " +
+                                "Notifying the thread to trigger a new rebalance immediately.", task.id());
+                        throw new TaskMigratedException(task);
+                    }
+
                     log.info("Reinitializing StandbyTask {}", task);
                     task.reinitializeStateStoresForPartitions(recoverableException.partitions());
                 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 62ddacfb33c81..9f02834dd7525 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -300,7 +300,6 @@ StreamTask activeTask(final TopicPartition partition) {
         return active.runningTaskFor(partition);
     }
 
-
     StandbyTask standbyTask(final TopicPartition partition) {
         return standby.runningTaskFor(partition);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
index 6c7b2b43c9d11..c4ea9647a8d72 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
@@ -61,6 +61,7 @@
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -78,18 +79,18 @@ public class EosIntegrationTest {
     });
 
     private static String applicationId;
+    private final static int NUM_TOPIC_PARTITIONS = 2;
     private final static String CONSUMER_GROUP_ID = "readCommitted";
     private final static String SINGLE_PARTITION_INPUT_TOPIC = "singlePartitionInputTopic";
     private final static String SINGLE_PARTITION_THROUGH_TOPIC = "singlePartitionThroughTopic";
     private final static String SINGLE_PARTITION_OUTPUT_TOPIC = "singlePartitionOutputTopic";
-    private final static int NUM_TOPIC_PARTITIONS = 2;
     private final static String MULTI_PARTITION_INPUT_TOPIC = "multiPartitionInputTopic";
     private final static String MULTI_PARTITION_THROUGH_TOPIC = "multiPartitionThroughTopic";
     private final static String MULTI_PARTITION_OUTPUT_TOPIC = "multiPartitionOutputTopic";
     private final String storeName = "store";
 
     private AtomicBoolean errorInjected;
-    private AtomicBoolean injectGC;
+    private AtomicBoolean gcInjected;
     private volatile boolean doGC = true;
     private AtomicInteger commitRequested;
     private Throwable uncaughtException;
@@ -153,7 +154,6 @@ private void runSimpleCopyTest(final int numberOfRestarts,
         output.to(outputTopic);
 
         for (int i = 0; i < numberOfRestarts; ++i) {
-            final long factor = i;
             final KafkaStreams streams = new KafkaStreams(
                 builder.build(),
                 StreamsTestUtils.getStreamsConfig(
@@ -171,7 +171,7 @@ private void runSimpleCopyTest(final int numberOfRestarts,
             try {
                 streams.start();
 
-                final List> inputData = prepareData(factor * 100, factor * 100 + 10L, 0L, 1L);
+                final List> inputData = prepareData(i * 100, i * 100 + 10L, 0L, 1L);
 
                 IntegrationTestUtils.produceKeyValuesSynchronously(
                     inputTopic,
@@ -510,7 +510,7 @@ public boolean conditionMet() {
             checkResultPerKey(committedRecords, committedDataBeforeGC);
             checkResultPerKey(uncommittedRecords, dataBeforeGC);
 
-            injectGC.set(true);
+            gcInjected.set(true);
             writeInputData(dataToTriggerFirstRebalance);
 
             TestUtils.waitForCondition(new TestCondition() {
@@ -577,7 +577,7 @@ private List> prepareData(final long fromInclusive, final l
     private KafkaStreams getKafkaStreams(final boolean withState, final String appDir, final int numberOfStreamsThreads) {
         commitRequested = new AtomicInteger(0);
         errorInjected = new AtomicBoolean(false);
-        injectGC = new AtomicBoolean(false);
+        gcInjected = new AtomicBoolean(false);
         final StreamsBuilder builder = new StreamsBuilder();
 
         String[] storeNames = null;
@@ -614,7 +614,7 @@ public KeyValue transform(final Long key, final Long value) {
                             // only tries to fail once on one of the task
                             throw new RuntimeException("Injected test exception.");
                         }
-                        if (injectGC.compareAndSet(true, false)) {
+                        if (gcInjected.compareAndSet(true, false)) {
                             while (doGC) {
                                 try {
                                     Thread.sleep(100);
@@ -779,6 +779,8 @@ private void verifyStateStore(final KafkaStreams streams, final Set it = store.all();
         while (it.hasNext()) {
             assertTrue(expectedStoreContent.remove(it.next()));

From 031f522a2d45c22540fb1e15ae0d0d3bf6061a98 Mon Sep 17 00:00:00 2001
From: Thomas Leplus 
Date: Tue, 27 Feb 2018 19:04:29 -0800
Subject: [PATCH 0105/1847] MINOR: Fix javadoc typo in Headers (#4627)

---
 .../src/main/java/org/apache/kafka/common/header/Headers.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/header/Headers.java b/clients/src/main/java/org/apache/kafka/common/header/Headers.java
index 1796d62944ae5..2353249cebcd3 100644
--- a/clients/src/main/java/org/apache/kafka/common/header/Headers.java
+++ b/clients/src/main/java/org/apache/kafka/common/header/Headers.java
@@ -41,7 +41,7 @@ public interface Headers extends Iterable
     {
      * Removes all headers for the given key returning if the operation succeeded.
      * 
      * @param key to remove all headers for.
-     * @return this instance of the Headers, once the header is added.
+     * @return this instance of the Headers, once the header is removed.
      * @throws IllegalStateException is thrown if headers are in a read-only state.
      */
     Headers remove(String key) throws IllegalStateException;

From 9166ae4ec45ae7dccaa90d8950b3583d7efd0a34 Mon Sep 17 00:00:00 2001
From: Viktor Somogyi 
Date: Wed, 28 Feb 2018 04:06:07 +0100
Subject: [PATCH 0106/1847] MINOR: Remove unnecessary semicolon in ResourceType
 (#4626)

---
 core/src/main/scala/kafka/security/auth/ResourceType.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/security/auth/ResourceType.scala b/core/src/main/scala/kafka/security/auth/ResourceType.scala
index 66da6108803d7..4ba5bcb26c67e 100644
--- a/core/src/main/scala/kafka/security/auth/ResourceType.scala
+++ b/core/src/main/scala/kafka/security/auth/ResourceType.scala
@@ -52,7 +52,7 @@ case object TransactionalId extends ResourceType {
 case object DelegationToken extends ResourceType {
   val name = "DelegationToken"
   val error = Errors.DELEGATION_TOKEN_AUTHORIZATION_FAILED
-  val toJava = JResourceType.DELEGATION_TOKEN;
+  val toJava = JResourceType.DELEGATION_TOKEN
 }
 
 object ResourceType {

From f582a7515c9303589a7cf964e53bd12992c66254 Mon Sep 17 00:00:00 2001
From: Ewen Cheslack-Postava 
Date: Wed, 28 Feb 2018 10:28:55 -0800
Subject: [PATCH 0107/1847] MINOR: Extend release.py with a subcommand for
 staging docs into the kafka-site repo

Author: Ewen Cheslack-Postava 

Reviewers: Ismael Juma , Damian Guy 

Closes #3917 from ewencp/stage-docs
---
 release.py | 164 +++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 135 insertions(+), 29 deletions(-)

diff --git a/release.py b/release.py
index b008794e733d8..0917184a44b5b 100755
--- a/release.py
+++ b/release.py
@@ -20,14 +20,31 @@
 """
 Utility for creating release candidates and promoting release candidates to a final relase.
 
-Usage: release.py
+Usage: release.py [subcommand]
 
-The utility is interactive; you will be prompted for basic release information and guided through the process.
+release.py stage
+
+  Builds and stages an RC for a release.
+
+  The utility is interactive; you will be prompted for basic release information and guided through the process.
+
+  This utility assumes you already have local a kafka git folder and that you
+  have added remotes corresponding to both:
+  (i) the github apache kafka mirror and
+  (ii) the apache kafka git repo.
+
+release.py stage-docs [kafka-site-path]
+
+  Builds the documentation and stages it into an instance of the Kafka website repository.
+
+  This is meant to automate the integration between the main Kafka website repository (https://github.com/apache/kafka-site)
+  and the versioned documentation maintained in the main Kafka repository. This is useful both for local testing and
+  development of docs (follow the instructions here: https://cwiki.apache.org/confluence/display/KAFKA/Setup+Kafka+Website+on+Local+Apache+Server)
+  as well as for committers to deploy docs (run this script, then validate, commit, and push to kafka-site).
+
+  With no arguments this script assumes you have the Kafka repository and kafka-site repository checked out side-by-side, but
+  you can specify a full path to the kafka-site repository if this is not the case.
 
-This utility assumes you already have local a kafka git folder and that you
-have added remotes corresponding to both:
-(i) the github apache kafka mirror and
-(ii) the apache kafka git repo.
 """
 
 from __future__ import print_function
@@ -145,11 +162,113 @@ def get_pref(prefs, name, request_fn):
         prefs[name] = val
     return val
 
-# Load saved preferences
-prefs = {}
-if os.path.exists(PREFS_FILE):
-    with open(PREFS_FILE, 'r') as prefs_fp:
-        prefs = json.load(prefs_fp)
+def load_prefs():
+    """Load saved preferences"""
+    prefs = {}
+    if os.path.exists(PREFS_FILE):
+        with open(PREFS_FILE, 'r') as prefs_fp:
+            prefs = json.load(prefs_fp)
+    return prefs
+
+def save_prefs(prefs):
+    """Save preferences"""
+    print("Saving preferences to %s" % PREFS_FILE)
+    with open(PREFS_FILE, 'w') as prefs_fp:
+        prefs = json.dump(prefs, prefs_fp)
+
+def get_jdk(prefs, version):
+    """
+    Get settings for the specified JDK version.
+    """
+    jdk_java_home = get_pref(prefs, 'jdk%d' % version, lambda: raw_input("Enter the path for JAVA_HOME for a JDK%d compiler (blank to use default JAVA_HOME): " % version))
+    jdk_env = dict(os.environ) if jdk_java_home.strip() else None
+    if jdk_env is not None: jdk_env['JAVA_HOME'] = jdk_java_home
+    if "1.%d.0" % version not in cmd_output("java -version", env=jdk_env):
+        fail("JDK %s is required" % version)
+    return jdk_env
+
+def get_version(repo=REPO_HOME):
+    """
+    Extracts the full version information as a str from gradle.properties
+    """
+    with open(os.path.join(repo, 'gradle.properties')) as fp:
+        for line in fp:
+            parts = line.split('=')
+            if parts[0].strip() != 'version': continue
+            return parts[1].strip()
+    fail("Couldn't extract version from gradle.properties")
+
+def docs_version(version):
+    """
+    Detects the major/minor version and converts it to the format used for docs on the website, e.g. gets 0.10.2.0-SNAPSHOT
+    from gradle.properties and converts it to 0102
+    """
+    version_parts = version.strip().split('.')
+    # 1.0+ will only have 3 version components as opposed to pre-1.0 that had 4
+    major_minor = version_parts[0:3] if version_parts[0] == '0' else version_parts[0:2]
+    return ''.join(major_minor)
+
+def docs_release_version(version):
+    """
+    Detects the version from gradle.properties and converts it to a release version number that should be valid for the
+    current release branch. For example, 0.10.2.0-SNAPSHOT would remain 0.10.2.0-SNAPSHOT (because no release has been
+    made on that branch yet); 0.10.2.1-SNAPSHOT would be converted to 0.10.2.0 because 0.10.2.1 is still in development
+    but 0.10.2.0 should have already been released. Regular version numbers (e.g. as encountered on a release branch)
+    will remain the same.
+    """
+    version_parts = version.strip().split('.')
+    if '-SNAPSHOT' in version_parts[-1]:
+        bugfix = int(version_parts[-1].split('-')[0])
+        if bugfix > 0:
+            version_parts[-1] = str(bugfix - 1)
+    return '.'.join(version_parts)
+
+def command_stage_docs():
+    kafka_site_repo_path = sys.argv[2] if len(sys.argv) > 2 else os.path.join(REPO_HOME, '..', 'kafka-site')
+    if not os.path.exists(kafka_site_repo_path) or not os.path.exists(os.path.join(kafka_site_repo_path, 'powered-by.html')):
+        sys.exit("%s doesn't exist or does not appear to be the kafka-site repository" % kafka_site_repo_path)
+
+    prefs = load_prefs()
+    jdk8_env = get_jdk(prefs, 8)
+    save_prefs(prefs)
+
+    version = get_version()
+    # We explicitly override the version of the project that we normally get from gradle.properties since we want to be
+    # able to run this from a release branch where we made some updates, but the build would show an incorrect SNAPSHOT
+    # version due to already having bumped the bugfix version number.
+    gradle_version_override = docs_release_version(version)
+
+    cmd("Building docs", "./gradlew -Pversion=%s clean releaseTarGzAll aggregatedJavadoc" % gradle_version_override, cwd=REPO_HOME, env=jdk8_env)
+
+    docs_tar = os.path.join(REPO_HOME, 'core', 'build', 'distributions', 'kafka_2.11-%s-site-docs.tgz' % gradle_version_override)
+
+    versioned_docs_path = os.path.join(kafka_site_repo_path, docs_version(version))
+    if not os.path.exists(versioned_docs_path):
+        os.mkdir(versioned_docs_path, 0755)
+
+    # The contents of the docs jar are site-docs/. We need to get rid of the site-docs prefix and dump everything
+    # inside it into the docs version subdirectory in the kafka-site repo
+    cmd('Extracting site-docs', 'tar xf %s --strip-components 1' % docs_tar, cwd=versioned_docs_path)
+
+    javadocs_src_dir = os.path.join(REPO_HOME, 'build', 'docs', 'javadoc')
+
+    cmd('Copying javadocs', 'cp -R %s %s' % (javadocs_src_dir, versioned_docs_path))
+
+    sys.exit(0)
+
+
+# Dispatch to subcommand
+subcommand = sys.argv[1] if len(sys.argv) > 1 else None
+if subcommand == 'stage-docs':
+    command_stage_docs()
+elif not (subcommand is None or subcommand == 'stage'):
+    fail("Unknown subcommand: %s" % subcommand)
+# else -> default subcommand stage
+
+
+## Default 'stage' subcommand implementation isn't isolated to its own function yet for historical reasons
+
+prefs = load_prefs()
 
 if not user_ok("""Requirements:
 1. Updated docs to reference the new release version where appropriate.
@@ -221,7 +340,7 @@ def get_pref(prefs, name, request_fn):
 rc = raw_input("Release candidate number: ")
 
 dev_branch = '.'.join(release_version_parts[:2])
-docs_version = ''.join(release_version_parts[:2])
+docs_release_version = docs_version(release_version[:2])
 
 # Validate that the release doesn't already exist and that the
 cmd("Fetching tags from upstream", 'git fetch --tags %s' % PUSH_REMOTE_NAME)
@@ -244,18 +363,8 @@ def get_pref(prefs, name, request_fn):
 # Prereq checks
 apache_id = get_pref(prefs, 'apache_id', lambda: raw_input("Enter your apache username: "))
 
-
-jdk7_java_home = get_pref(prefs, 'jdk7', lambda: raw_input("Enter the path for JAVA_HOME for a JDK7 compiler (blank to use default JAVA_HOME): "))
-jdk7_env = dict(os.environ) if jdk7_java_home.strip() else None
-if jdk7_env is not None: jdk7_env['JAVA_HOME'] = jdk7_java_home
-if "1.7.0" not in cmd_output("java -version", env=jdk7_env):
-    fail("You must be able to build artifacts with JDK7 for Scala 2.10 and 2.11 artifacts")
-
-jdk8_java_home = get_pref(prefs, 'jdk8', lambda: raw_input("Enter the path for JAVA_HOME for a JDK8 compiler (blank to use default JAVA_HOME): "))
-jdk8_env = dict(os.environ) if jdk8_java_home.strip() else None
-if jdk8_env is not None: jdk8_env['JAVA_HOME'] = jdk8_java_home
-if "1.8.0" not in cmd_output("java -version", env=jdk8_env):
-    fail("You must be able to build artifacts with JDK8 for Scala 2.12 artifacts")
+jdk7_env = get_jdk(prefs, 7)
+jdk8_env = get_jdk(prefs, 8)
 
 
 def select_gpg_key():
@@ -275,10 +384,7 @@ def select_gpg_key():
     gpg_test_tempfile.write("abcdefg")
     cmd("Testing GPG key & passphrase", ["gpg", "--batch", "--pinentry-mode", "loopback", "--passphrase-fd", "0", "-u", key_name, "--armor", "--output", gpg_test_tempfile.name + ".asc", "--detach-sig", gpg_test_tempfile.name], stdin=gpg_passphrase)
 
-# Save preferences
-print("Saving preferences to %s" % PREFS_FILE)
-with open(PREFS_FILE, 'w') as prefs_fp:
-    prefs = json.dump(prefs, prefs_fp)
+save_prefs(prefs)
 
 # Generate RC
 try:
@@ -400,7 +506,7 @@ def select_gpg_key():
                                'rc_tag': rc_tag,
                                'rc_githash': rc_githash,
                                'dev_branch': dev_branch,
-                               'docs_version': docs_version,
+                               'docs_version': docs_release_version,
                                'apache_id': apache_id,
                                }
 

From eb449fe7c55a0816328a851fc1102dfeac6d8616 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 1 Mar 2018 09:27:11 -0800
Subject: [PATCH 0108/1847] KAFKA-6560: Replace range query with newly added
 single point query in Windowed Aggregation (#4578)

* Add a new fetch(K key, long window-start-timestamp) API into ReadOnlyWindowStore.
* Use the new API to replace the range fetch API in KStreamWindowedAggregate and KStreamWindowedReduce.
* Added corresponding unit tests.
* Also removed some redundant byte serdes in byte stores.
---
 .../internals/KStreamWindowAggregate.java     |  61 +--
 .../internals/KStreamWindowReduce.java        |  65 +--
 .../streams/state/ReadOnlyWindowStore.java    |  24 +-
 .../state/internals/CachingKeyValueStore.java |   8 +-
 .../state/internals/CachingWindowStore.java   |  20 +-
 .../ChangeLoggingWindowBytesStore.java        |  13 +-
 .../CompositeReadOnlyWindowStore.java         |  23 +-
 .../state/internals/MeteredWindowStore.java   |  16 +-
 .../state/internals/RocksDBWindowStore.java   |  16 +-
 .../state/internals/WindowKeySchema.java      |  10 +-
 .../kafka/streams/state/NoOpWindowStore.java  |   5 +
 .../internals/CachingWindowStoreTest.java     |   5 +
 .../CompositeReadOnlyWindowStoreTest.java     |  18 +-
 .../internals/ReadOnlyWindowStoreStub.java    |  11 +-
 .../internals/RocksDBWindowStoreTest.java     | 415 +++++++++---------
 15 files changed, 377 insertions(+), 333 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
index ec26866bd22db..27f8408320f4d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
@@ -17,7 +17,6 @@
 package org.apache.kafka.streams.kstream.internals;
 
 import org.apache.kafka.streams.kstream.Aggregator;
-import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.Initializer;
 import org.apache.kafka.streams.kstream.Windows;
 import org.apache.kafka.streams.kstream.Window;
@@ -39,7 +38,10 @@ public class KStreamWindowAggregate implements KStrea
 
     private boolean sendOldValues = false;
 
-    public KStreamWindowAggregate(Windows windows, String storeName, Initializer initializer, Aggregator aggregator) {
+    KStreamWindowAggregate(final Windows windows,
+                           final String storeName,
+                           final Initializer initializer,
+                           final Aggregator aggregator) {
         this.windows = windows;
         this.storeName = storeName;
         this.initializer = initializer;
@@ -63,7 +65,7 @@ private class KStreamWindowAggregateProcessor extends AbstractProcessor {
 
         @SuppressWarnings("unchecked")
         @Override
-        public void init(ProcessorContext context) {
+        public void init(final ProcessorContext context) {
             super.init(context);
 
             windowStore = (WindowStore) context.getStateStore(storeName);
@@ -71,54 +73,27 @@ public void init(ProcessorContext context) {
         }
 
         @Override
-        public void process(K key, V value) {
+        public void process(final K key, final V value) {
             // if the key is null, we do not need proceed aggregating the record
             // the record with the table
             if (key == null)
                 return;
 
             // first get the matching windows
-            long timestamp = context().timestamp();
-            Map matchedWindows = windows.windowsFor(timestamp);
+            final long timestamp = context().timestamp();
+            final Map matchedWindows = windows.windowsFor(timestamp);
 
-            long timeFrom = Long.MAX_VALUE;
-            long timeTo = Long.MIN_VALUE;
+            // try update the window, and create the new window for the rest of unmatched window that do not exist yet
+            for (final Map.Entry entry : matchedWindows.entrySet()) {
+                T oldAgg = windowStore.fetch(key, entry.getKey());
 
-            // use range query on window store for efficient reads
-            for (long windowStartMs : matchedWindows.keySet()) {
-                timeFrom = windowStartMs < timeFrom ? windowStartMs : timeFrom;
-                timeTo = windowStartMs > timeTo ? windowStartMs : timeTo;
-            }
-
-            try (WindowStoreIterator iter = windowStore.fetch(key, timeFrom, timeTo)) {
-
-                // for each matching window, try to update the corresponding key
-                while (iter.hasNext()) {
-                    KeyValue entry = iter.next();
-                    W window = matchedWindows.get(entry.key);
-
-                    if (window != null) {
-
-                        T oldAgg = entry.value;
-
-                        if (oldAgg == null)
-                            oldAgg = initializer.apply();
-
-                        // try to add the new value (there will never be old value)
-                        T newAgg = aggregator.apply(key, value, oldAgg);
-
-                        // update the store with the new value
-                        windowStore.put(key, newAgg, window.start());
-                        tupleForwarder.maybeForward(new Windowed<>(key, window), newAgg, oldAgg);
-                        matchedWindows.remove(entry.key);
-                    }
+                if (oldAgg == null) {
+                    oldAgg = initializer.apply();
                 }
-            }
 
-            // create the new window for the rest of unmatched window that do not exist yet
-            for (Map.Entry entry : matchedWindows.entrySet()) {
-                T oldAgg = initializer.apply();
-                T newAgg = aggregator.apply(key, value, oldAgg);
+                final T newAgg = aggregator.apply(key, value, oldAgg);
+
+                // update the store with the new value
                 windowStore.put(key, newAgg, entry.getKey());
                 tupleForwarder.maybeForward(new Windowed<>(key, entry.getValue()), newAgg, oldAgg);
             }
@@ -147,13 +122,13 @@ private class KStreamWindowAggregateValueGetter implements KTableValueGetter) context.getStateStore(storeName);
         }
 
         @SuppressWarnings("unchecked")
         @Override
-        public T get(Windowed windowedKey) {
+        public T get(final Windowed windowedKey) {
             K key = windowedKey.key();
             W window = (W) windowedKey.window();
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
index 7d02f118f891e..c3d95d81b86c9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
-import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.Window;
 import org.apache.kafka.streams.kstream.Windowed;
@@ -37,7 +36,9 @@ public class KStreamWindowReduce implements KStreamAggPr
 
     private boolean sendOldValues = false;
 
-    public KStreamWindowReduce(Windows windows, String storeName, Reducer reducer) {
+    KStreamWindowReduce(final Windows windows,
+                        final String storeName,
+                        final Reducer reducer) {
         this.windows = windows;
         this.storeName = storeName;
         this.reducer = reducer;
@@ -60,63 +61,37 @@ private class KStreamWindowReduceProcessor extends AbstractProcessor {
 
         @SuppressWarnings("unchecked")
         @Override
-        public void init(ProcessorContext context) {
+        public void init(final ProcessorContext context) {
             super.init(context);
             windowStore = (WindowStore) context.getStateStore(storeName);
             tupleForwarder = new TupleForwarder<>(windowStore, context, new ForwardingCacheFlushListener, V>(context, sendOldValues), sendOldValues);
         }
 
         @Override
-        public void process(K key, V value) {
+        public void process(final K key, final V value) {
             // if the key is null, we do not need proceed aggregating
             // the record with the table
             if (key == null)
                 return;
 
             // first get the matching windows
-            long timestamp = context().timestamp();
+            final long timestamp = context().timestamp();
+            final Map matchedWindows = windows.windowsFor(timestamp);
 
-            Map matchedWindows = windows.windowsFor(timestamp);
-
-            long timeFrom = Long.MAX_VALUE;
-            long timeTo = Long.MIN_VALUE;
-
-            // use range query on window store for efficient reads
-            for (long windowStartMs : matchedWindows.keySet()) {
-                timeFrom = windowStartMs < timeFrom ? windowStartMs : timeFrom;
-                timeTo = windowStartMs > timeTo ? windowStartMs : timeTo;
-            }
+            // try update the window, and create the new window for the rest of unmatched window that do not exist yet
+            for (final Map.Entry entry : matchedWindows.entrySet()) {
+                final V oldAgg = windowStore.fetch(key, entry.getKey());
 
-            try (WindowStoreIterator iter = windowStore.fetch(key, timeFrom, timeTo)) {
-                // for each matching window, try to update the corresponding key and send to the downstream
-                while (iter.hasNext()) {
-                    KeyValue entry = iter.next();
-                    W window = matchedWindows.get(entry.key);
-
-                    if (window != null) {
-
-                        V oldAgg = entry.value;
-                        V newAgg = oldAgg;
-
-                        // try to add the new value (there will never be old value)
-                        if (newAgg == null) {
-                            newAgg = value;
-                        } else {
-                            newAgg = reducer.apply(newAgg, value);
-                        }
-
-                        // update the store with the new value
-                        windowStore.put(key, newAgg, window.start());
-                        tupleForwarder.maybeForward(new Windowed<>(key, window), newAgg, oldAgg);
-                        matchedWindows.remove(entry.key);
-                    }
+                V newAgg;
+                if (oldAgg == null) {
+                    newAgg = value;
+                } else {
+                    newAgg = reducer.apply(oldAgg, value);
                 }
-            }
 
-            // create the new window for the rest of unmatched window that do not exist yet
-            for (final Map.Entry entry : matchedWindows.entrySet()) {
-                windowStore.put(key, value, entry.getKey());
-                tupleForwarder.maybeForward(new Windowed<>(key, entry.getValue()), value, null);
+                // update the store with the new value
+                windowStore.put(key, newAgg, entry.getKey());
+                tupleForwarder.maybeForward(new Windowed<>(key, entry.getValue()), newAgg, oldAgg);
             }
         }
     }
@@ -143,13 +118,13 @@ private class KStreamWindowReduceValueGetter implements KTableValueGetter) context.getStateStore(storeName);
         }
 
         @SuppressWarnings("unchecked")
         @Override
-        public V get(Windowed windowedKey) {
+        public V get(final Windowed windowedKey) {
             K key = windowedKey.key();
             W window = (W) windowedKey.window();
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyWindowStore.java
index f92ab6eea1589..dea759f486c30 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyWindowStore.java
@@ -28,6 +28,17 @@
  */
 public interface ReadOnlyWindowStore {
 
+    /**
+     * Get the value of key from a window.
+     *
+     * @param key       the key to fetch
+     * @param time      start timestamp (inclusive) of the window
+     * @return The value or {@code null} if no value is found in the window
+     * @throws InvalidStateStoreException if the store is not initialized
+     * @throws NullPointerException If {@code null} is used for any key.
+     */
+    V fetch(K key, long time);
+
     /**
      * Get all the key-value pairs with the given key and the time range from all
      * the existing windows.
@@ -56,9 +67,12 @@ public interface ReadOnlyWindowStore {
      * For each key, the iterator guarantees ordering of windows, starting from the oldest/earliest
      * available window to the newest/latest window.
      *
+     * @param key       the key to fetch
+     * @param timeFrom  time range start (inclusive)
+     * @param timeTo    time range end (inclusive)
      * @return an iterator over key-value pairs {@code }
      * @throws InvalidStateStoreException if the store is not initialized
-     * @throws NullPointerException If null is used for key.
+     * @throws NullPointerException If {@code null} is used for key.
      */
     WindowStoreIterator fetch(K key, long timeFrom, long timeTo);
 
@@ -74,7 +88,7 @@ public interface ReadOnlyWindowStore {
      * @param timeTo    time range end (inclusive)
      * @return an iterator over windowed key-value pairs {@code , value>}
      * @throws InvalidStateStoreException if the store is not initialized
-     * @throws NullPointerException If null is used for any key.
+     * @throws NullPointerException If {@code null} is used for any key.
      */
     KeyValueIterator, V> fetch(K from, K to, long timeFrom, long timeTo);
     
@@ -89,11 +103,11 @@ public interface ReadOnlyWindowStore {
     /**
      * Gets all the key-value pairs that belong to the windows within in the given time range.
      *
-     * @param timeFrom the beginning of the time slot from which to search
-     * @param timeTo   the end of the time slot from which to search
+     * @param timeFrom the beginning of the time slot from which to search (inclusive)
+     * @param timeTo   the end of the time slot from which to search (inclusive)
      * @return an iterator over windowed key-value pairs {@code , value>}
      * @throws InvalidStateStoreException if the store is not initialized
-     * @throws NullPointerException if null is used for any key
+     * @throws NullPointerException if {@code null} is used for any key
      */
     KeyValueIterator, V> fetchAll(long timeFrom, long timeTo);
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
index 82a6ac785a8eb..45f606f375e65 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
@@ -175,13 +175,9 @@ private byte[] getInternal(final Bytes key) {
                 cache.put(cacheName, key, new LRUCacheEntry(rawValue));
             }
             return rawValue;
+        } else {
+            return entry.value;
         }
-
-        if (entry.value == null) {
-            return null;
-        }
-
-        return entry.value;
     }
 
     @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
index ad0bd99ecb47b..e3d0f62930618 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
@@ -45,11 +45,12 @@ class CachingWindowStore extends WrappedStateStore.AbstractStateStore impl
 
     private String name;
     private ThreadCache cache;
-    private InternalProcessorContext context;
+    private boolean sendOldValues;
     private StateSerdes serdes;
+    private InternalProcessorContext context;
     private StateSerdes bytesSerdes;
     private CacheFlushListener, V> flushListener;
-    private boolean sendOldValues;
+
     private final SegmentedCacheFunction cacheFunction;
 
     CachingWindowStore(final WindowStore underlying,
@@ -150,12 +151,25 @@ public synchronized void put(final Bytes key, final byte[] value, final long tim
         // if store is open outside as well.
         validateStoreOpen();
         
-        final Bytes keyBytes = WindowStoreUtils.toBinaryKey(key, timestamp, 0, bytesSerdes);
+        final Bytes keyBytes = WindowStoreUtils.toBinaryKey(key.get(), timestamp, 0);
         final LRUCacheEntry entry = new LRUCacheEntry(value, true, context.offset(),
                                                       timestamp, context.partition(), context.topic());
         cache.put(name, cacheFunction.cacheKey(keyBytes), entry);
     }
 
+    @Override
+    public byte[] fetch(final Bytes key, final long timestamp) {
+        validateStoreOpen();
+        final Bytes bytesKey = WindowStoreUtils.toBinaryKey(key.get(), timestamp, 0);
+        final Bytes cacheKey = cacheFunction.cacheKey(bytesKey);
+        final LRUCacheEntry entry = cache.get(name, cacheKey);
+        if (entry == null) {
+            return underlying.fetch(key, timestamp);
+        } else {
+            return entry.value;
+        }
+    }
+
     @Override
     public synchronized WindowStoreIterator fetch(final Bytes key, final long timeFrom, final long timeTo) {
         // since this function may not access the underlying inner store, we need to validate
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java
index 4fe4b99112bcc..e69a320e5102e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingWindowBytesStore.java
@@ -36,7 +36,6 @@ class ChangeLoggingWindowBytesStore extends WrappedStateStore.AbstractStateStore
     private final boolean retainDuplicates;
     private StoreChangeLogger changeLogger;
     private ProcessorContext context;
-    private StateSerdes innerStateSerde;
     private int seqnum = 0;
 
     ChangeLoggingWindowBytesStore(final WindowStore bytesStore,
@@ -46,6 +45,11 @@ class ChangeLoggingWindowBytesStore extends WrappedStateStore.AbstractStateStore
         this.retainDuplicates = retainDuplicates;
     }
 
+    @Override
+    public byte[] fetch(final Bytes key, final long timestamp) {
+        return bytesStore.fetch(key, timestamp);
+    }
+
     @Override
     public WindowStoreIterator fetch(final Bytes key, final long from, final long to) {
         return bytesStore.fetch(key, from, to);
@@ -74,18 +78,19 @@ public void put(final Bytes key, final byte[] value) {
     @Override
     public void put(final Bytes key, final byte[] value, final long timestamp) {
         bytesStore.put(key, value, timestamp);
-        changeLogger.logChange(WindowStoreUtils.toBinaryKey(key, timestamp, maybeUpdateSeqnumForDups(), innerStateSerde), value);
+        changeLogger.logChange(WindowStoreUtils.toBinaryKey(key.get(), timestamp, maybeUpdateSeqnumForDups()), value);
     }
 
     @Override
     public void init(final ProcessorContext context, final StateStore root) {
         this.context = context;
         bytesStore.init(context, root);
-        innerStateSerde = WindowStoreUtils.getInnerStateSerde(ProcessorStateManager.storeChangelogTopic(context.applicationId(), bytesStore.name()));
+
+        final StateSerdes bytesSerde = WindowStoreUtils.getInnerStateSerde(ProcessorStateManager.storeChangelogTopic(context.applicationId(), bytesStore.name()));
         changeLogger = new StoreChangeLogger<>(
             name(),
             context,
-            innerStateSerde);
+            bytesSerde);
     }
 
     private int maybeUpdateSeqnumForDups() {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java
index 6afc6fdaba617..1b5d5e5611a9b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStore.java
@@ -44,11 +44,30 @@ public CompositeReadOnlyWindowStore(final StateStoreProvider provider,
         this.storeName = storeName;
     }
 
+    @Override
+    public V fetch(final K key, final long time) {
+        Objects.requireNonNull(key, "key can't be null");
+        final List> stores = provider.stores(storeName, windowStoreType);
+        for (final ReadOnlyWindowStore windowStore : stores) {
+            try {
+                final V result = windowStore.fetch(key, time);
+                if (result != null) {
+                    return result;
+                }
+            } catch (final InvalidStateStoreException e) {
+                throw new InvalidStateStoreException(
+                        "State store is not available anymore and may have been migrated to another instance; " +
+                                "please re-discover its location from the state metadata.");
+            }
+        }
+        return null;
+    }
+
     @Override
     public WindowStoreIterator fetch(final K key, final long timeFrom, final long timeTo) {
         Objects.requireNonNull(key, "key can't be null");
         final List> stores = provider.stores(storeName, windowStoreType);
-        for (ReadOnlyWindowStore windowStore : stores) {
+        for (final ReadOnlyWindowStore windowStore : stores) {
             try {
                 final WindowStoreIterator result = windowStore.fetch(key, timeFrom, timeTo);
                 if (!result.hasNext()) {
@@ -56,7 +75,7 @@ public WindowStoreIterator fetch(final K key, final long timeFrom, final long
                 } else {
                     return result;
                 }
-            } catch (InvalidStateStoreException e) {
+            } catch (final InvalidStateStoreException e) {
                 throw new InvalidStateStoreException(
                         "State store is not available anymore and may have been migrated to another instance; " +
                                 "please re-discover its location from the state metadata.");
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
index e8900052b978c..15961e7c72184 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java
@@ -91,7 +91,7 @@ public void put(final K key, final V value) {
 
     @Override
     public void put(final K key, final V value, final long timestamp) {
-        long startNs = time.nanoseconds();
+        final long startNs = time.nanoseconds();
         try {
             inner.put(keyBytes(key), serdes.rawValue(value), timestamp);
         } finally {
@@ -103,6 +103,20 @@ private Bytes keyBytes(final K key) {
         return Bytes.wrap(serdes.rawKey(key));
     }
 
+    @Override
+    public V fetch(final K key, final long timestamp) {
+        final long startNs = time.nanoseconds();
+        V ret;
+        try {
+            final byte[] result = inner.fetch(keyBytes(key), timestamp);
+            ret = serdes.valueFrom(result);
+        } finally {
+            metrics.recordLatency(this.fetchTime, startNs, time.nanoseconds());
+        }
+
+        return ret;
+    }
+
     @Override
     public WindowStoreIterator fetch(final K key, final long timeFrom, final long timeTo) {
         return new MeteredWindowStoreIterator<>(inner.fetch(keyBytes(key), timeFrom, timeTo),
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
index f1a9c63af8b3f..58c345a7867c1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
@@ -96,25 +96,31 @@ public void init(final ProcessorContext context, final StateStore root) {
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(final K key, final V value) {
         put(key, value, context.timestamp());
     }
 
     @Override
-    public void put(K key, V value, long timestamp) {
+    public void put(final K key, final V value, final long timestamp) {
         maybeUpdateSeqnumForDups();
 
         bytesStore.put(WindowStoreUtils.toBinaryKey(key, timestamp, seqnum, serdes), serdes.rawValue(value));
     }
 
     @Override
-    public WindowStoreIterator fetch(K key, long timeFrom, long timeTo) {
+    public V fetch(final K key, final long timestamp) {
+        final byte[] bytesValue = bytesStore.get(WindowStoreUtils.toBinaryKey(key, timestamp, seqnum, serdes));
+        return serdes.valueFrom(bytesValue);
+    }
+
+    @Override
+    public WindowStoreIterator fetch(final K key, final long timeFrom, final long timeTo) {
         final KeyValueIterator bytesIterator = bytesStore.fetch(Bytes.wrap(serdes.rawKey(key)), timeFrom, timeTo);
         return new WindowStoreIteratorWrapper<>(bytesIterator, serdes, windowSize).valuesIterator();
     }
 
     @Override
-    public KeyValueIterator, V> fetch(K from, K to, long timeFrom, long timeTo) {
+    public KeyValueIterator, V> fetch(final K from, final K to, final long timeFrom, final long timeTo) {
         final KeyValueIterator bytesIterator = bytesStore.fetch(Bytes.wrap(serdes.rawKey(from)), Bytes.wrap(serdes.rawKey(to)), timeFrom, timeTo);
         return new WindowStoreIteratorWrapper<>(bytesIterator, serdes, windowSize).keyValueIterator();
     }
@@ -126,7 +132,7 @@ public KeyValueIterator, V> all() {
     }
     
     @Override
-    public KeyValueIterator, V> fetchAll(long timeFrom, long timeTo) {
+    public KeyValueIterator, V> fetchAll(final long timeFrom, final long timeTo) {
         final KeyValueIterator bytesIterator = bytesStore.fetchAll(timeFrom, timeTo);
         return new WindowStoreIteratorWrapper<>(bytesIterator, serdes, windowSize).keyValueIterator();
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowKeySchema.java
index 739792f1877ec..e432baad691ad 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowKeySchema.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowKeySchema.java
@@ -16,10 +16,8 @@
  */
 package org.apache.kafka.streams.state.internals;
 
-import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.state.KeyValueIterator;
-import org.apache.kafka.streams.state.StateSerdes;
 
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -29,11 +27,9 @@ class WindowKeySchema implements RocksDBSegmentedBytesStore.KeySchema {
     private static final int SUFFIX_SIZE = WindowStoreUtils.TIMESTAMP_SIZE + WindowStoreUtils.SEQNUM_SIZE;
     private static final byte[] MIN_SUFFIX = new byte[SUFFIX_SIZE];
 
-    private StateSerdes serdes;
-
     @Override
     public void init(final String topic) {
-        serdes = new StateSerdes<>(topic, Serdes.Bytes(), Serdes.ByteArray());
+        // nothing to do
     }
 
     @Override
@@ -53,12 +49,12 @@ public Bytes lowerRange(final Bytes key, final long from) {
 
     @Override
     public Bytes lowerRangeFixedSize(final Bytes key, final long from) {
-        return WindowStoreUtils.toBinaryKey(key, Math.max(0, from), 0, serdes);
+        return WindowStoreUtils.toBinaryKey(key.get(), Math.max(0, from), 0);
     }
 
     @Override
     public Bytes upperRangeFixedSize(final Bytes key, final long to) {
-        return WindowStoreUtils.toBinaryKey(key, to, Integer.MAX_VALUE, serdes);
+        return WindowStoreUtils.toBinaryKey(key.get(), to, Integer.MAX_VALUE);
     }
 
     @Override
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java b/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java
index 1ded31fcc3559..05016bc8c6900 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/NoOpWindowStore.java
@@ -82,6 +82,11 @@ public boolean isOpen() {
         return false;
     }
 
+    @Override
+    public Object fetch(final Object key, final long time) {
+        return null;
+    }
+
     @Override
     public WindowStoreIterator fetch(final Object key, final long timeFrom, final long timeTo) {
         return EMPTY_WINDOW_STORE_ITERATOR;
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
index 239a007918d02..5f934b83454f6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
@@ -96,6 +96,11 @@ public void shouldPutFetchFromCache() {
         cachingStore.put(bytesKey("a"), bytesValue("a"));
         cachingStore.put(bytesKey("b"), bytesValue("b"));
 
+        assertThat(cachingStore.fetch(bytesKey("a"), 10), equalTo(bytesValue("a")));
+        assertThat(cachingStore.fetch(bytesKey("b"), 10), equalTo(bytesValue("b")));
+        assertThat(cachingStore.fetch(bytesKey("c"), 10), equalTo(null));
+        assertThat(cachingStore.fetch(bytesKey("a"), 0), equalTo(null));
+
         final WindowStoreIterator a = cachingStore.fetch(bytesKey("a"), 10, 10);
         final WindowStoreIterator b = cachingStore.fetch(bytesKey("b"), 10, 10);
         verifyKeyValue(a.next(), DEFAULT_TIMESTAMP, "a");
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStoreTest.java
index 58fddaa248da0..a241510e6e5ae 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyWindowStoreTest.java
@@ -169,8 +169,7 @@ public void emptyIteratorNextShouldThrowNoSuchElementException() {
 
     @Test
     public void shouldFetchKeyRangeAcrossStores() {
-        final ReadOnlyWindowStoreStub secondUnderlying = new
-                ReadOnlyWindowStoreStub<>(WINDOW_SIZE);
+        final ReadOnlyWindowStoreStub secondUnderlying = new ReadOnlyWindowStoreStub<>(WINDOW_SIZE);
         stubProviderTwo.addStore(storeName, secondUnderlying);
         underlyingWindowStore.put("a", "a", 0L);
         secondUnderlying.put("b", "b", 10L);
@@ -179,7 +178,20 @@ public void shouldFetchKeyRangeAcrossStores() {
                 KeyValue.pair(new Windowed<>("a", new TimeWindow(0, WINDOW_SIZE)), "a"),
                 KeyValue.pair(new Windowed<>("b", new TimeWindow(10, 10 + WINDOW_SIZE)), "b"))));
     }
-    
+
+    @Test
+    public void shouldFetchKeyValueAcrossStores() {
+        final ReadOnlyWindowStoreStub secondUnderlyingWindowStore = new ReadOnlyWindowStoreStub<>(WINDOW_SIZE);
+        stubProviderTwo.addStore(storeName, secondUnderlyingWindowStore);
+        underlyingWindowStore.put("a", "a", 0L);
+        secondUnderlyingWindowStore.put("b", "b", 10L);
+        assertThat(windowStore.fetch("a", 0L), equalTo("a"));
+        assertThat(windowStore.fetch("b", 10L), equalTo("b"));
+        assertThat(windowStore.fetch("c", 10L), equalTo(null));
+        assertThat(windowStore.fetch("a", 10L), equalTo(null));
+    }
+
+
     @Test
     public void shouldGetAllAcrossStores() {
         final ReadOnlyWindowStoreStub secondUnderlying = new
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java
index 256df33a5a6fa..6d911a39486d0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ReadOnlyWindowStoreStub.java
@@ -47,7 +47,16 @@ public class ReadOnlyWindowStoreStub implements ReadOnlyWindowStore,
     public ReadOnlyWindowStoreStub(long windowSize) {
         this.windowSize = windowSize;
     }
-    
+
+    @Override
+    public V fetch(final K key, final long time) {
+        final Map kvMap = data.get(time);
+        if (kvMap != null) {
+            return kvMap.get(key);
+        } else {
+            return null;
+        }
+    }
 
     @Override
     public WindowStoreIterator fetch(final K key, final long timeFrom, final long timeTo) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
index 39c8f0360afc2..f7572987f39a8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.streams.processor.internals.RecordCollector;
 import org.apache.kafka.streams.processor.internals.RecordCollectorImpl;
 import org.apache.kafka.streams.state.StateSerdes;
+import org.apache.kafka.streams.state.Stores;
 import org.apache.kafka.streams.state.WindowStore;
 import org.apache.kafka.streams.state.WindowStoreIterator;
 import org.apache.kafka.test.MockProcessorContext;
@@ -46,7 +47,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -66,7 +66,7 @@ public class RocksDBWindowStoreTest {
     private static final long DEFAULT_CACHE_SIZE_BYTES = 1024 * 1024L;
 
     private final int numSegments = 3;
-    private final static long WINDOW_SIZE = 3;
+    private final long windowSize = 3L;
     private final String windowName = "window";
     private final long segmentSize = Segments.MIN_SEGMENT_INTERVAL;
     private final long retentionPeriod = segmentSize * (numSegments - 1);
@@ -95,27 +95,37 @@ public  void send(final String topic,
 
     private final File baseDir = TestUtils.tempDirectory("test");
     private final MockProcessorContext context = new MockProcessorContext(baseDir, Serdes.ByteArray(), Serdes.ByteArray(), recordCollector, cache);
-    private WindowStore windowStore;
+    private WindowStore windowStore;
+
+    private WindowStore createWindowStore(final ProcessorContext context, final boolean retainDuplicates) {
+        final WindowStore store = Stores.windowStoreBuilder(
+                Stores.persistentWindowStore(windowName,
+                        retentionPeriod,
+                        numSegments,
+                        windowSize,
+                        retainDuplicates),
+                Serdes.Integer(),
+                Serdes.String()).build();
 
-    @SuppressWarnings("unchecked")
-    private  WindowStore createWindowStore(ProcessorContext context, final boolean enableCaching, final boolean retainDuplicates) {
-        final RocksDBWindowStoreSupplier supplier = new RocksDBWindowStoreSupplier<>(windowName, retentionPeriod, numSegments, retainDuplicates, Serdes.Integer(), Serdes.String(),
-                                                                                     WINDOW_SIZE, true, Collections.emptyMap(), enableCaching);
-        final WindowStore store = (WindowStore) supplier.get();
         store.init(context, store);
         return store;
     }
 
+    private WindowStore createWindowStore(final ProcessorContext context) {
+        return createWindowStore(context, false);
+    }
+
     @After
     public void closeStore() {
         context.close();
-        windowStore.close();
+        if (windowStore != null) {
+            windowStore.close();
+        }
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void shouldOnlyIterateOpenSegments() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         long currentTime = 0;
         context.setRecordContext(createRecordContext(currentTime));
         windowStore.put(1, "one");
@@ -145,38 +155,50 @@ private ProcessorRecordContext createRecordContext(final long time) {
         return new ProcessorRecordContext(time, 0, 0, "topic");
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testPutAndFetch() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testRangeAndSinglePointFetch() {
+        windowStore = createWindowStore(context);
         long startTime = segmentSize - 4L;
 
         putFirstBatch(windowStore, startTime, context);
 
-        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime + 0L - WINDOW_SIZE, startTime + 0L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + 1L - WINDOW_SIZE, startTime + 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L - WINDOW_SIZE, startTime + 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + 3L - WINDOW_SIZE, startTime + 3L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + 4L - WINDOW_SIZE, startTime + 4L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + 5L - WINDOW_SIZE, startTime + 5L + WINDOW_SIZE)));
+        assertEquals("zero", windowStore.fetch(0, startTime));
+        assertEquals("one", windowStore.fetch(1, startTime + 1L));
+        assertEquals("two", windowStore.fetch(2, startTime + 2L));
+        assertEquals("four", windowStore.fetch(4, startTime + 4L));
+        assertEquals("five", windowStore.fetch(5, startTime + 5L));
+
+        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime + 0L - windowSize, startTime + 0L + windowSize)));
+        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + 1L - windowSize, startTime + 1L + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L - windowSize, startTime + 2L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + 3L - windowSize, startTime + 3L + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + 4L - windowSize, startTime + 4L + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + 5L - windowSize, startTime + 5L + windowSize)));
 
         putSecondBatch(windowStore, startTime, context);
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime - 2L - WINDOW_SIZE, startTime - 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime - 1L - WINDOW_SIZE, startTime - 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1"), toList(windowStore.fetch(2, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2"), toList(windowStore.fetch(2, startTime + 1L - WINDOW_SIZE, startTime + 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3"), toList(windowStore.fetch(2, startTime + 2L - WINDOW_SIZE, startTime + 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3", "two+4"), toList(windowStore.fetch(2, startTime + 3L - WINDOW_SIZE, startTime + 3L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3", "two+4", "two+5"), toList(windowStore.fetch(2, startTime + 4L - WINDOW_SIZE, startTime + 4L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 5L - WINDOW_SIZE, startTime + 5L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+1", "two+2", "two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 6L - WINDOW_SIZE, startTime + 6L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+2", "two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 7L - WINDOW_SIZE, startTime + 7L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 8L - WINDOW_SIZE, startTime + 8L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 9L - WINDOW_SIZE, startTime + 9L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+5", "two+6"), toList(windowStore.fetch(2, startTime + 10L - WINDOW_SIZE, startTime + 10L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+6"), toList(windowStore.fetch(2, startTime + 11L - WINDOW_SIZE, startTime + 11L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 12L - WINDOW_SIZE, startTime + 12L + WINDOW_SIZE)));
+        assertEquals("two+1", windowStore.fetch(2, startTime + 3L));
+        assertEquals("two+2", windowStore.fetch(2, startTime + 4L));
+        assertEquals("two+3", windowStore.fetch(2, startTime + 5L));
+        assertEquals("two+4", windowStore.fetch(2, startTime + 6L));
+        assertEquals("two+5", windowStore.fetch(2, startTime + 7L));
+        assertEquals("two+6", windowStore.fetch(2, startTime + 8L));
+
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime - 2L - windowSize, startTime - 2L + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime - 1L - windowSize, startTime - 1L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1"), toList(windowStore.fetch(2, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2"), toList(windowStore.fetch(2, startTime + 1L - windowSize, startTime + 1L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3"), toList(windowStore.fetch(2, startTime + 2L - windowSize, startTime + 2L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3", "two+4"), toList(windowStore.fetch(2, startTime + 3L - windowSize, startTime + 3L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3", "two+4", "two+5"), toList(windowStore.fetch(2, startTime + 4L - windowSize, startTime + 4L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 5L - windowSize, startTime + 5L + windowSize)));
+        assertEquals(Utils.mkList("two+1", "two+2", "two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 6L - windowSize, startTime + 6L + windowSize)));
+        assertEquals(Utils.mkList("two+2", "two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 7L - windowSize, startTime + 7L + windowSize)));
+        assertEquals(Utils.mkList("two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 8L - windowSize, startTime + 8L + windowSize)));
+        assertEquals(Utils.mkList("two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 9L - windowSize, startTime + 9L + windowSize)));
+        assertEquals(Utils.mkList("two+5", "two+6"), toList(windowStore.fetch(2, startTime + 10L - windowSize, startTime + 10L + windowSize)));
+        assertEquals(Utils.mkList("two+6"), toList(windowStore.fetch(2, startTime + 11L - windowSize, startTime + 11L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 12L - windowSize, startTime + 12L + windowSize)));
 
         // Flush the store and verify all current entries were properly flushed ...
         windowStore.flush();
@@ -192,10 +214,9 @@ public void testPutAndFetch() throws IOException {
         assertNull(entriesByKey.get(6));
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void shouldGetAll() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void shouldGetAll() {
+        windowStore = createWindowStore(context);
         long startTime = segmentSize - 4L;
 
         putFirstBatch(windowStore, startTime, context);
@@ -212,10 +233,9 @@ public void shouldGetAll() throws IOException {
         );
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void shouldFetchAllInTimeRange() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void shouldFetchAllInTimeRange() {
+        windowStore = createWindowStore(context);
         long startTime = segmentSize - 4L;
 
         putFirstBatch(windowStore, startTime, context);
@@ -242,10 +262,9 @@ public void shouldFetchAllInTimeRange() throws IOException {
         );
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testFetchRange() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testFetchRange() {
+        windowStore = createWindowStore(context);
         long startTime = segmentSize - 4L;
 
         putFirstBatch(windowStore, startTime, context);
@@ -258,71 +277,70 @@ public void testFetchRange() throws IOException {
 
         assertEquals(
             Utils.mkList(zero, one),
-            StreamsTestUtils.toList(windowStore.fetch(0, 1, startTime + 0L - WINDOW_SIZE, startTime + 0L + WINDOW_SIZE))
+            StreamsTestUtils.toList(windowStore.fetch(0, 1, startTime + 0L - windowSize, startTime + 0L + windowSize))
         );
         assertEquals(
             Utils.mkList(one),
-            StreamsTestUtils.toList(windowStore.fetch(1, 1, startTime + 0L - WINDOW_SIZE, startTime + 0L + WINDOW_SIZE))
+            StreamsTestUtils.toList(windowStore.fetch(1, 1, startTime + 0L - windowSize, startTime + 0L + windowSize))
         );
         assertEquals(
             Utils.mkList(one, two),
-            StreamsTestUtils.toList(windowStore.fetch(1, 3, startTime + 0L - WINDOW_SIZE, startTime + 0L + WINDOW_SIZE))
+            StreamsTestUtils.toList(windowStore.fetch(1, 3, startTime + 0L - windowSize, startTime + 0L + windowSize))
         );
         assertEquals(
             Utils.mkList(zero, one, two),
-            StreamsTestUtils.toList(windowStore.fetch(0, 5, startTime + 0L - WINDOW_SIZE, startTime + 0L + WINDOW_SIZE))
+            StreamsTestUtils.toList(windowStore.fetch(0, 5, startTime + 0L - windowSize, startTime + 0L + windowSize))
         );
         assertEquals(
             Utils.mkList(zero, one, two,
                          four, five),
-            StreamsTestUtils.toList(windowStore.fetch(0, 5, startTime + 0L - WINDOW_SIZE, startTime + 0L + WINDOW_SIZE + 5L))
+            StreamsTestUtils.toList(windowStore.fetch(0, 5, startTime + 0L - windowSize, startTime + 0L + windowSize + 5L))
         );
         assertEquals(
             Utils.mkList(two, four, five),
-            StreamsTestUtils.toList(windowStore.fetch(0, 5, startTime + 2L, startTime + 0L + WINDOW_SIZE + 5L))
+            StreamsTestUtils.toList(windowStore.fetch(0, 5, startTime + 2L, startTime + 0L + windowSize + 5L))
         );
         assertEquals(
             Utils.mkList(),
-            StreamsTestUtils.toList(windowStore.fetch(4, 5, startTime + 2L, startTime + WINDOW_SIZE))
+            StreamsTestUtils.toList(windowStore.fetch(4, 5, startTime + 2L, startTime + windowSize))
         );
         assertEquals(
             Utils.mkList(),
-            StreamsTestUtils.toList(windowStore.fetch(0, 3, startTime + 3L, startTime + WINDOW_SIZE + 5))
+            StreamsTestUtils.toList(windowStore.fetch(0, 3, startTime + 3L, startTime + windowSize + 5))
         );
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testPutAndFetchBefore() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testPutAndFetchBefore() {
+        windowStore = createWindowStore(context);
         long startTime = segmentSize - 4L;
 
         putFirstBatch(windowStore, startTime, context);
 
-        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime + 0L - WINDOW_SIZE, startTime + 0L)));
-        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + 1L - WINDOW_SIZE, startTime + 1L)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L - WINDOW_SIZE, startTime + 2L)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + 3L - WINDOW_SIZE, startTime + 3L)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + 4L - WINDOW_SIZE, startTime + 4L)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + 5L - WINDOW_SIZE, startTime + 5L)));
+        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime + 0L - windowSize, startTime + 0L)));
+        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + 1L - windowSize, startTime + 1L)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L - windowSize, startTime + 2L)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + 3L - windowSize, startTime + 3L)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + 4L - windowSize, startTime + 4L)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + 5L - windowSize, startTime + 5L)));
 
         putSecondBatch(windowStore, startTime, context);
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime - 1L - WINDOW_SIZE, startTime - 1L)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 0L - WINDOW_SIZE, startTime + 0L)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 1L - WINDOW_SIZE, startTime + 1L)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L - WINDOW_SIZE, startTime + 2L)));
-        assertEquals(Utils.mkList("two", "two+1"), toList(windowStore.fetch(2, startTime + 3L - WINDOW_SIZE, startTime + 3L)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2"), toList(windowStore.fetch(2, startTime + 4L - WINDOW_SIZE, startTime + 4L)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3"), toList(windowStore.fetch(2, startTime + 5L - WINDOW_SIZE, startTime + 5L)));
-        assertEquals(Utils.mkList("two+1", "two+2", "two+3", "two+4"), toList(windowStore.fetch(2, startTime + 6L - WINDOW_SIZE, startTime + 6L)));
-        assertEquals(Utils.mkList("two+2", "two+3", "two+4", "two+5"), toList(windowStore.fetch(2, startTime + 7L - WINDOW_SIZE, startTime + 7L)));
-        assertEquals(Utils.mkList("two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 8L - WINDOW_SIZE, startTime + 8L)));
-        assertEquals(Utils.mkList("two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 9L - WINDOW_SIZE, startTime + 9L)));
-        assertEquals(Utils.mkList("two+5", "two+6"), toList(windowStore.fetch(2, startTime + 10L - WINDOW_SIZE, startTime + 10L)));
-        assertEquals(Utils.mkList("two+6"), toList(windowStore.fetch(2, startTime + 11L - WINDOW_SIZE, startTime + 11L)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 12L - WINDOW_SIZE, startTime + 12L)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 13L - WINDOW_SIZE, startTime + 13L)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime - 1L - windowSize, startTime - 1L)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 0L - windowSize, startTime + 0L)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 1L - windowSize, startTime + 1L)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L - windowSize, startTime + 2L)));
+        assertEquals(Utils.mkList("two", "two+1"), toList(windowStore.fetch(2, startTime + 3L - windowSize, startTime + 3L)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2"), toList(windowStore.fetch(2, startTime + 4L - windowSize, startTime + 4L)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3"), toList(windowStore.fetch(2, startTime + 5L - windowSize, startTime + 5L)));
+        assertEquals(Utils.mkList("two+1", "two+2", "two+3", "two+4"), toList(windowStore.fetch(2, startTime + 6L - windowSize, startTime + 6L)));
+        assertEquals(Utils.mkList("two+2", "two+3", "two+4", "two+5"), toList(windowStore.fetch(2, startTime + 7L - windowSize, startTime + 7L)));
+        assertEquals(Utils.mkList("two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 8L - windowSize, startTime + 8L)));
+        assertEquals(Utils.mkList("two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 9L - windowSize, startTime + 9L)));
+        assertEquals(Utils.mkList("two+5", "two+6"), toList(windowStore.fetch(2, startTime + 10L - windowSize, startTime + 10L)));
+        assertEquals(Utils.mkList("two+6"), toList(windowStore.fetch(2, startTime + 11L - windowSize, startTime + 11L)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 12L - windowSize, startTime + 12L)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 13L - windowSize, startTime + 13L)));
 
         // Flush the store and verify all current entries were properly flushed ...
         windowStore.flush();
@@ -338,38 +356,37 @@ public void testPutAndFetchBefore() throws IOException {
         assertNull(entriesByKey.get(6));
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testPutAndFetchAfter() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testPutAndFetchAfter() {
+        windowStore = createWindowStore(context);
         long startTime = segmentSize - 4L;
 
         putFirstBatch(windowStore, startTime, context);
 
-        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime + 0L, startTime + 0L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + 1L, startTime + 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L, startTime + 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + 3L, startTime + 3L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + 4L, startTime + 4L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + 5L, startTime + 5L + WINDOW_SIZE)));
+        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime + 0L, startTime + 0L + windowSize)));
+        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + 1L, startTime + 1L + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + 2L, startTime + 2L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + 3L, startTime + 3L + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + 4L, startTime + 4L + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + 5L, startTime + 5L + windowSize)));
 
         putSecondBatch(windowStore, startTime, context);
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime - 2L, startTime - 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime - 1L, startTime - 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1"), toList(windowStore.fetch(2, startTime, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2"), toList(windowStore.fetch(2, startTime + 1L, startTime + 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3"), toList(windowStore.fetch(2, startTime + 2L, startTime + 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+1", "two+2", "two+3", "two+4"), toList(windowStore.fetch(2, startTime + 3L, startTime + 3L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+2", "two+3", "two+4", "two+5"), toList(windowStore.fetch(2, startTime + 4L, startTime + 4L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 5L, startTime + 5L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 6L, startTime + 6L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+5", "two+6"), toList(windowStore.fetch(2, startTime + 7L, startTime + 7L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two+6"), toList(windowStore.fetch(2, startTime + 8L, startTime + 8L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 9L, startTime + 9L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 10L, startTime + 10L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 11L, startTime + 11L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 12L, startTime + 12L + WINDOW_SIZE)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime - 2L, startTime - 2L + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime - 1L, startTime - 1L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1"), toList(windowStore.fetch(2, startTime, startTime + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2"), toList(windowStore.fetch(2, startTime + 1L, startTime + 1L + windowSize)));
+        assertEquals(Utils.mkList("two", "two+1", "two+2", "two+3"), toList(windowStore.fetch(2, startTime + 2L, startTime + 2L + windowSize)));
+        assertEquals(Utils.mkList("two+1", "two+2", "two+3", "two+4"), toList(windowStore.fetch(2, startTime + 3L, startTime + 3L + windowSize)));
+        assertEquals(Utils.mkList("two+2", "two+3", "two+4", "two+5"), toList(windowStore.fetch(2, startTime + 4L, startTime + 4L + windowSize)));
+        assertEquals(Utils.mkList("two+3", "two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 5L, startTime + 5L + windowSize)));
+        assertEquals(Utils.mkList("two+4", "two+5", "two+6"), toList(windowStore.fetch(2, startTime + 6L, startTime + 6L + windowSize)));
+        assertEquals(Utils.mkList("two+5", "two+6"), toList(windowStore.fetch(2, startTime + 7L, startTime + 7L + windowSize)));
+        assertEquals(Utils.mkList("two+6"), toList(windowStore.fetch(2, startTime + 8L, startTime + 8L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 9L, startTime + 9L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 10L, startTime + 10L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 11L, startTime + 11L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + 12L, startTime + 12L + windowSize)));
 
         // Flush the store and verify all current entries were properly flushed ...
         windowStore.flush();
@@ -385,26 +402,25 @@ public void testPutAndFetchAfter() throws IOException {
         assertNull(entriesByKey.get(6));
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testPutSameKeyTimestamp() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testPutSameKeyTimestamp() {
+        windowStore = createWindowStore(context, true);
         long startTime = segmentSize - 4L;
 
         context.setRecordContext(createRecordContext(startTime));
         windowStore.put(0, "zero");
 
-        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
+        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
 
         windowStore.put(0, "zero");
         windowStore.put(0, "zero+");
         windowStore.put(0, "zero++");
 
-        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime + 1L - WINDOW_SIZE, startTime + 1L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime + 2L - WINDOW_SIZE, startTime + 2L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime + 3L - WINDOW_SIZE, startTime + 3L + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime + 4L - WINDOW_SIZE, startTime + 4L + WINDOW_SIZE)));
+        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime + 1L - windowSize, startTime + 1L + windowSize)));
+        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime + 2L - windowSize, startTime + 2L + windowSize)));
+        assertEquals(Utils.mkList("zero", "zero", "zero+", "zero++"), toList(windowStore.fetch(0, startTime + 3L - windowSize, startTime + 3L + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime + 4L - windowSize, startTime + 4L + windowSize)));
 
         // Flush the store and verify all current entries were properly flushed ...
         windowStore.flush();
@@ -414,11 +430,9 @@ public void testPutSameKeyTimestamp() throws IOException {
         assertEquals(Utils.mkSet("zero@0", "zero@0", "zero+@0", "zero++@0"), entriesByKey.get(0));
     }
 
-
-    @SuppressWarnings("unchecked")
     @Test
-    public void testRolling() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testRolling() {
+        windowStore = createWindowStore(context);
 
         // to validate segments
         final Segments segments = new Segments(windowName, retentionPeriod, numSegments);
@@ -450,12 +464,12 @@ public void testRolling() throws IOException {
                                  segments.segmentName(3),
                                  segments.segmentName(4)), segmentDirs(baseDir));
 
-        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + incr - WINDOW_SIZE, startTime + incr + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + incr * 2 - WINDOW_SIZE, startTime + incr * 2 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - WINDOW_SIZE, startTime + incr * 3 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - WINDOW_SIZE, startTime + incr * 4 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - WINDOW_SIZE, startTime + incr * 5 + WINDOW_SIZE)));
+        assertEquals(Utils.mkList("zero"), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList("one"), toList(windowStore.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - windowSize, startTime + incr * 3 + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - windowSize, startTime + incr * 4 + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - windowSize, startTime + incr * 5 + windowSize)));
 
         context.setRecordContext(createRecordContext(startTime + incr * 6));
         windowStore.put(6, "six");
@@ -464,13 +478,13 @@ public void testRolling() throws IOException {
                                  segments.segmentName(5)), segmentDirs(baseDir));
 
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - WINDOW_SIZE, startTime + incr + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + incr * 2 - WINDOW_SIZE, startTime + incr * 2 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - WINDOW_SIZE, startTime + incr * 3 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - WINDOW_SIZE, startTime + incr * 4 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - WINDOW_SIZE, startTime + incr * 5 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - WINDOW_SIZE, startTime + incr * 6 + WINDOW_SIZE)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - windowSize, startTime + incr * 3 + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - windowSize, startTime + incr * 4 + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - windowSize, startTime + incr * 5 + windowSize)));
+        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - windowSize, startTime + incr * 6 + windowSize)));
 
 
         context.setRecordContext(createRecordContext(startTime + incr * 7));
@@ -479,14 +493,14 @@ public void testRolling() throws IOException {
                                  segments.segmentName(4),
                                  segments.segmentName(5)), segmentDirs(baseDir));
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - WINDOW_SIZE, startTime + incr + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + incr * 2 - WINDOW_SIZE, startTime + incr * 2 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - WINDOW_SIZE, startTime + incr * 3 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - WINDOW_SIZE, startTime + incr * 4 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - WINDOW_SIZE, startTime + incr * 5 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - WINDOW_SIZE, startTime + incr * 6 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("seven"), toList(windowStore.fetch(7, startTime + incr * 7 - WINDOW_SIZE, startTime + incr * 7 + WINDOW_SIZE)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
+        assertEquals(Utils.mkList("two"), toList(windowStore.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - windowSize, startTime + incr * 3 + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - windowSize, startTime + incr * 4 + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - windowSize, startTime + incr * 5 + windowSize)));
+        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - windowSize, startTime + incr * 6 + windowSize)));
+        assertEquals(Utils.mkList("seven"), toList(windowStore.fetch(7, startTime + incr * 7 - windowSize, startTime + incr * 7 + windowSize)));
 
         context.setRecordContext(createRecordContext(startTime + incr * 8));
         windowStore.put(8, "eight");
@@ -495,15 +509,15 @@ public void testRolling() throws IOException {
                                  segments.segmentName(6)), segmentDirs(baseDir));
 
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - WINDOW_SIZE, startTime + incr + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + incr * 2 - WINDOW_SIZE, startTime + incr * 2 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - WINDOW_SIZE, startTime + incr * 3 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - WINDOW_SIZE, startTime + incr * 4 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - WINDOW_SIZE, startTime + incr * 5 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - WINDOW_SIZE, startTime + incr * 6 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("seven"), toList(windowStore.fetch(7, startTime + incr * 7 - WINDOW_SIZE, startTime + incr * 7 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("eight"), toList(windowStore.fetch(8, startTime + incr * 8 - WINDOW_SIZE, startTime + incr * 8 + WINDOW_SIZE)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - windowSize, startTime + incr * 3 + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - windowSize, startTime + incr * 4 + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - windowSize, startTime + incr * 5 + windowSize)));
+        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - windowSize, startTime + incr * 6 + windowSize)));
+        assertEquals(Utils.mkList("seven"), toList(windowStore.fetch(7, startTime + incr * 7 - windowSize, startTime + incr * 7 + windowSize)));
+        assertEquals(Utils.mkList("eight"), toList(windowStore.fetch(8, startTime + incr * 8 - windowSize, startTime + incr * 8 + windowSize)));
 
         // check segment directories
         windowStore.flush();
@@ -514,14 +528,12 @@ public void testRolling() throws IOException {
 
     }
 
-
-    @SuppressWarnings("unchecked")
     @Test
     public void testRestore() throws IOException {
         long startTime = segmentSize * 2;
         long incr = segmentSize / 2;
 
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         context.setRecordContext(createRecordContext(startTime));
         windowStore.put(0, "zero");
         context.setRecordContext(createRecordContext(startTime + incr));
@@ -547,28 +559,28 @@ public void testRestore() throws IOException {
         // remove local store image
         Utils.delete(baseDir);
 
-        windowStore = createWindowStore(context, false, true);
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - WINDOW_SIZE, startTime + incr + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + incr * 2 - WINDOW_SIZE, startTime + incr * 2 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - WINDOW_SIZE, startTime + incr * 3 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(4, startTime + incr * 4 - WINDOW_SIZE, startTime + incr * 4 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(5, startTime + incr * 5 - WINDOW_SIZE, startTime + incr * 5 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(6, startTime + incr * 6 - WINDOW_SIZE, startTime + incr * 6 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(7, startTime + incr * 7 - WINDOW_SIZE, startTime + incr * 7 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(8, startTime + incr * 8 - WINDOW_SIZE, startTime + incr * 8 + WINDOW_SIZE)));
+        windowStore = createWindowStore(context);
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - windowSize, startTime + incr * 3 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(4, startTime + incr * 4 - windowSize, startTime + incr * 4 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(5, startTime + incr * 5 - windowSize, startTime + incr * 5 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(6, startTime + incr * 6 - windowSize, startTime + incr * 6 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(7, startTime + incr * 7 - windowSize, startTime + incr * 7 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(8, startTime + incr * 8 - windowSize, startTime + incr * 8 + windowSize)));
 
         context.restore(windowName, changeLog);
 
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - WINDOW_SIZE, startTime + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - WINDOW_SIZE, startTime + incr + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + incr * 2 - WINDOW_SIZE, startTime + incr * 2 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - WINDOW_SIZE, startTime + incr * 3 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - WINDOW_SIZE, startTime + incr * 4 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - WINDOW_SIZE, startTime + incr * 5 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - WINDOW_SIZE, startTime + incr * 6 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("seven"), toList(windowStore.fetch(7, startTime + incr * 7 - WINDOW_SIZE, startTime + incr * 7 + WINDOW_SIZE)));
-        assertEquals(Utils.mkList("eight"), toList(windowStore.fetch(8, startTime + incr * 8 - WINDOW_SIZE, startTime + incr * 8 + WINDOW_SIZE)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(0, startTime - windowSize, startTime + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
+        assertEquals(Utils.mkList(), toList(windowStore.fetch(3, startTime + incr * 3 - windowSize, startTime + incr * 3 + windowSize)));
+        assertEquals(Utils.mkList("four"), toList(windowStore.fetch(4, startTime + incr * 4 - windowSize, startTime + incr * 4 + windowSize)));
+        assertEquals(Utils.mkList("five"), toList(windowStore.fetch(5, startTime + incr * 5 - windowSize, startTime + incr * 5 + windowSize)));
+        assertEquals(Utils.mkList("six"), toList(windowStore.fetch(6, startTime + incr * 6 - windowSize, startTime + incr * 6 + windowSize)));
+        assertEquals(Utils.mkList("seven"), toList(windowStore.fetch(7, startTime + incr * 7 - windowSize, startTime + incr * 7 + windowSize)));
+        assertEquals(Utils.mkList("eight"), toList(windowStore.fetch(8, startTime + incr * 8 - windowSize, startTime + incr * 8 + windowSize)));
 
         // check segment directories
         windowStore.flush();
@@ -578,10 +590,9 @@ public void testRestore() throws IOException {
         );
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testSegmentMaintenance() throws IOException {
-        windowStore = createWindowStore(context, false, true);
+    public void testSegmentMaintenance() {
+        windowStore = createWindowStore(context, true);
         context.setTime(0L);
         context.setRecordContext(createRecordContext(0));
         windowStore.put(0, "v");
@@ -655,12 +666,11 @@ public void testSegmentMaintenance() throws IOException {
 
     }
 
-    @SuppressWarnings("unchecked")
     @Test
-    public void testInitialLoading() throws IOException {
+    public void testInitialLoading() {
         File storeDir = new File(baseDir, windowName);
 
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
 
         new File(storeDir, segments.segmentName(0L)).mkdir();
         new File(storeDir, segments.segmentName(1L)).mkdir();
@@ -671,7 +681,7 @@ public void testInitialLoading() throws IOException {
         new File(storeDir, segments.segmentName(6L)).mkdir();
         windowStore.close();
 
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
 
         assertEquals(
                 Utils.mkSet(segments.segmentName(4L), segments.segmentName(5L), segments.segmentName(6L)),
@@ -690,10 +700,9 @@ public void testInitialLoading() throws IOException {
         );
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void shouldCloseOpenIteratorsWhenStoreIsClosedAndThrowInvalidStateStoreExceptionOnHasNextAndNext() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         context.setRecordContext(createRecordContext(0));
         windowStore.put(1, "one", 1L);
         windowStore.put(1, "two", 2L);
@@ -717,24 +726,19 @@ public void shouldCloseOpenIteratorsWhenStoreIsClosedAndThrowInvalidStateStoreEx
         }
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void shouldFetchAndIterateOverExactKeys() {
         final long windowSize = 0x7a00000000000000L;
         final long retentionPeriod = 0x7a00000000000000L;
-        final RocksDBWindowStoreSupplier supplier =
-                new RocksDBWindowStoreSupplier<>(
-                    "window",
-                    retentionPeriod, 2,
-                    true,
-                    Serdes.String(),
-                    Serdes.String(),
-                    windowSize,
-                    true,
-                    Collections.emptyMap(),
-                    false);
-
-        windowStore = supplier.get();
+        final WindowStore windowStore = Stores.windowStoreBuilder(
+                Stores.persistentWindowStore(windowName,
+                        retentionPeriod,
+                        2,
+                        windowSize,
+                        true),
+                Serdes.String(),
+                Serdes.String()).build();
+
         windowStore.init(context, windowStore);
 
         windowStore.put("a", "0001", 0);
@@ -763,50 +767,45 @@ public void shouldFetchAndIterateOverExactKeys() {
 
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerExceptionOnPutNullKey() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         windowStore.put(null, "anyValue");
     }
 
     @Test
     public void shouldNotThrowNullPointerExceptionOnPutNullValue() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         windowStore.put(1, null);
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerExceptionOnGetNullKey() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         windowStore.fetch(null, 1L, 2L);
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerExceptionOnRangeNullFromKey() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         windowStore.fetch(null, 2, 1L, 2L);
     }
 
     @Test(expected = NullPointerException.class)
     public void shouldThrowNullPointerExceptionOnRangeNullToKey() {
-        windowStore = createWindowStore(context, false, true);
+        windowStore = createWindowStore(context);
         windowStore.fetch(1, null, 1L, 2L);
     }
 
-    @SuppressWarnings("unchecked")
     @Test
     public void shouldFetchAndIterateOverExactBinaryKeys() {
-        final RocksDBWindowStoreSupplier supplier =
-                new RocksDBWindowStoreSupplier<>(
-                        "window",
-                        60000, 2,
-                        true,
-                        Serdes.Bytes(),
-                        Serdes.String(),
+        final WindowStore windowStore = Stores.windowStoreBuilder(
+                Stores.persistentWindowStore(windowName,
+                        60000,
+                        2,
                         60000,
-                        true,
-                        Collections.emptyMap(),
-                        false);
+                        true),
+                Serdes.Bytes(),
+                Serdes.String()).build();
 
-        windowStore = supplier.get();
         windowStore.init(context, windowStore);
 
         final Bytes key1 = Bytes.wrap(new byte[]{0});
@@ -891,11 +890,11 @@ private Map> entriesByKey(List> ch
         return entriesByKey;
     }
 
-    private static  KeyValue, V> windowedPair(K key, V value, long timestamp) {
-        return windowedPair(key, value, timestamp, WINDOW_SIZE);
+    private  KeyValue, V> windowedPair(K key, V value, long timestamp) {
+        return windowedPair(key, value, timestamp, windowSize);
     }
 
-    private static  KeyValue, V> windowedPair(K key, V value, long timestamp, long windowSize) {
+    private  KeyValue, V> windowedPair(K key, V value, long timestamp, long windowSize) {
         return KeyValue.pair(new Windowed<>(key, WindowStoreUtils.timeWindowForSize(timestamp, windowSize)), value);
     }
 }

From 6cfcc9d553622e7d511a849935e9b504f947399d Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Thu, 1 Mar 2018 11:04:11 -0800
Subject: [PATCH 0109/1847] KAFKA-6593; Fix livelock with consumer heartbeat
 thread in commitSync (#4625)

Contention for the lock in ConsumerNetworkClient can lead to a livelock situation in which an active commitSync is unable to make progress because its completion is blocked in the heartbeat thread. The fix is twofold:

1) We change ConsumerNetworkClient to use a fair lock to reduce the chance of each thread getting starved.
2) We eliminate the dependence on the lock in ConsumerNetworkClient for callback completion so that callbacks will not be blocked by an active poll().

Reviewers: Guozhang Wang 
---
 .../internals/AbstractCoordinator.java        | 48 +++++----
 .../internals/ConsumerCoordinator.java        |  8 +-
 .../internals/ConsumerNetworkClient.java      | 99 ++++++++++++++-----
 .../org/apache/kafka/clients/MockClient.java  | 38 ++++++-
 .../internals/ConsumerCoordinatorTest.java    | 11 ++-
 .../internals/ConsumerNetworkClientTest.java  | 66 ++++++++++++-
 6 files changed, 213 insertions(+), 57 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index b39f52c055219..c3edaa77534ee 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -231,7 +231,7 @@ protected synchronized boolean ensureCoordinatorReady(long startTimeMs, long tim
             } else if (coordinator != null && client.connectionFailed(coordinator)) {
                 // we found the coordinator, but the connection has failed, so mark
                 // it dead and backoff before retrying discovery
-                coordinatorDead();
+                markCoordinatorUnknown();
                 time.sleep(retryBackoffMs);
             }
 
@@ -487,7 +487,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
             } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                     || error == Errors.NOT_COORDINATOR) {
                 // re-discover the coordinator and retry with backoff
-                coordinatorDead();
+                markCoordinatorUnknown();
                 log.debug("Attempt to join group failed due to obsolete coordinator information: {}", error.message());
                 future.raise(error);
             } else if (error == Errors.INCONSISTENT_GROUP_PROTOCOL
@@ -550,7 +550,7 @@ public void handle(SyncGroupResponse syncResponse,
                 if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
                     future.raise(new GroupAuthorizationException(groupId));
                 } else if (error == Errors.REBALANCE_IN_PROGRESS) {
-                    log.debug("SyncGroup failed due to group rebalance");
+                    log.debug("SyncGroup failed because the group began another rebalance");
                     future.raise(error);
                 } else if (error == Errors.UNKNOWN_MEMBER_ID
                         || error == Errors.ILLEGAL_GENERATION) {
@@ -559,8 +559,8 @@ public void handle(SyncGroupResponse syncResponse,
                     future.raise(error);
                 } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                         || error == Errors.NOT_COORDINATOR) {
-                    log.debug("SyncGroup failed:", error.message());
-                    coordinatorDead();
+                    log.debug("SyncGroup failed: {}", error.message());
+                    markCoordinatorUnknown();
                     future.raise(error);
                 } else {
                     future.raise(new KafkaException("Unexpected error from SyncGroup: " + error.message()));
@@ -627,27 +627,34 @@ public void onFailure(RuntimeException e, RequestFuture future) {
      * @return true if the coordinator is unknown
      */
     public boolean coordinatorUnknown() {
-        return coordinator() == null;
+        return checkAndGetCoordinator() == null;
     }
 
     /**
-     * Get the current coordinator
+     * Get the coordinator if its connection is still active. Otherwise mark it unknown and
+     * return null.
+     *
      * @return the current coordinator or null if it is unknown
      */
-    protected synchronized Node coordinator() {
+    protected synchronized Node checkAndGetCoordinator() {
         if (coordinator != null && client.connectionFailed(coordinator)) {
-            coordinatorDead();
+            markCoordinatorUnknown(true);
             return null;
         }
         return this.coordinator;
     }
 
-    /**
-     * Mark the current coordinator as dead.
-     */
-    protected synchronized void coordinatorDead() {
+    private synchronized Node coordinator() {
+        return this.coordinator;
+    }
+
+    protected synchronized void markCoordinatorUnknown() {
+        markCoordinatorUnknown(false);
+    }
+
+    protected synchronized void markCoordinatorUnknown(boolean isDisconnected) {
         if (this.coordinator != null) {
-            log.info("Marking the coordinator {} dead", this.coordinator);
+            log.info("Group coordinator {} is unavailable or invalid, will attempt rediscovery", this.coordinator);
             Node oldCoordinator = this.coordinator;
 
             // Mark the coordinator dead before disconnecting requests since the callbacks for any pending
@@ -656,8 +663,9 @@ protected synchronized void coordinatorDead() {
             this.coordinator = null;
 
             // Disconnect from the coordinator to ensure that there are no in-flight requests remaining.
-            // Pending callbacks will be invoked with a DisconnectException.
-            client.disconnect(oldCoordinator);
+            // Pending callbacks will be invoked with a DisconnectException on the next call to poll.
+            if (!isDisconnected)
+                client.disconnectAsync(oldCoordinator);
         }
     }
 
@@ -708,7 +716,7 @@ protected void close(long timeoutMs) {
                 // interrupted using wakeup) and the leave group request which have been queued, but not
                 // yet sent to the broker. Wait up to close timeout for these pending requests to be processed.
                 // If coordinator is not known, requests are aborted.
-                Node coordinator = coordinator();
+                Node coordinator = checkAndGetCoordinator();
                 if (coordinator != null && !client.awaitPendingRequests(coordinator, timeoutMs))
                     log.warn("Close timed out with {} pending requests to coordinator, terminating client connections",
                             client.pendingRequestCount(coordinator));
@@ -769,7 +777,7 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
                     || error == Errors.NOT_COORDINATOR) {
                 log.info("Attempt to heartbeat failed since coordinator {} is either not started or not valid.",
                         coordinator());
-                coordinatorDead();
+                markCoordinatorUnknown();
                 future.raise(error);
             } else if (error == Errors.REBALANCE_IN_PROGRESS) {
                 log.info("Attempt to heartbeat failed since group is rebalancing");
@@ -800,7 +808,7 @@ protected abstract class CoordinatorResponseHandler extends RequestFutureA
         public void onFailure(RuntimeException e, RequestFuture future) {
             // mark the coordinator as dead
             if (e instanceof DisconnectException) {
-                coordinatorDead();
+                markCoordinatorUnknown(true);
             }
             future.raise(e);
         }
@@ -948,7 +956,7 @@ public void run() {
                         } else if (heartbeat.sessionTimeoutExpired(now)) {
                             // the session timeout has expired without seeing a successful heartbeat, so we should
                             // probably make sure the coordinator is still healthy.
-                            coordinatorDead();
+                            markCoordinatorUnknown();
                         } else if (heartbeat.pollTimeoutExpired(now)) {
                             // the poll timeout has expired, which means that the foreground thread has stalled
                             // in between calls to poll(), so we explicitly leave the group.
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 0aa61bbf442f0..2afa1ff923685 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -684,7 +684,7 @@ private RequestFuture sendOffsetCommitRequest(final Map futu
                     } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                             || error == Errors.NOT_COORDINATOR
                             || error == Errors.REQUEST_TIMED_OUT) {
-                        coordinatorDead();
+                        markCoordinatorUnknown();
                         future.raise(error);
                         return;
                     } else if (error == Errors.UNKNOWN_MEMBER_ID
@@ -799,7 +799,7 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu
      * @return A request future containing the committed offsets.
      */
     private RequestFuture> sendOffsetFetchRequest(Set partitions) {
-        Node coordinator = coordinator();
+        Node coordinator = checkAndGetCoordinator();
         if (coordinator == null)
             return RequestFuture.coordinatorNotAvailable();
 
@@ -825,7 +825,7 @@ public void handle(OffsetFetchResponse response, RequestFuture pendingCompletion = new ConcurrentLinkedQueue<>();
 
+    private final ConcurrentLinkedQueue pendingDisconnects = new ConcurrentLinkedQueue<>();
+
     // this flag allows the client to be safely woken up without waiting on the lock above. It is
     // atomic to avoid the need to acquire the lock above in order to enable it concurrently.
     private final AtomicBoolean wakeup = new AtomicBoolean(false);
@@ -113,12 +119,22 @@ public RequestFuture send(Node node, AbstractRequest.Builder
         return completionHandler.future;
     }
 
-    public synchronized Node leastLoadedNode() {
-        return client.leastLoadedNode(time.milliseconds());
+    public Node leastLoadedNode() {
+        lock.lock();
+        try {
+            return client.leastLoadedNode(time.milliseconds());
+        } finally {
+            lock.unlock();
+        }
     }
 
-    public synchronized boolean hasReadyNodes() {
-        return client.hasReadyNodes();
+    public boolean hasReadyNodes() {
+        lock.lock();
+        try {
+            return client.hasReadyNodes();
+        } finally {
+            lock.unlock();
+        }
     }
 
     /**
@@ -227,14 +243,18 @@ public void poll(long timeout, long now, PollCondition pollCondition, boolean di
         // there may be handlers which need to be invoked if we woke up the previous call to poll
         firePendingCompletedRequests();
 
-        synchronized (this) {
+        lock.lock();
+        try {
+            // Handle async disconnects prior to attempting any sends
+            handlePendingDisconnects();
+
             // send all the requests we can send now
             trySend(now);
 
             // check whether the poll is still needed by the caller. Note that if the expected completion
             // condition becomes satisfied after the call to shouldBlock() (because of a fired completion
             // handler), the client will be woken up.
-            if (pollCondition == null || pollCondition.shouldBlock()) {
+            if (pendingCompletion.isEmpty() && (pollCondition == null || pollCondition.shouldBlock())) {
                 // if there are no requests in flight, do not block longer than the retry backoff
                 if (client.inFlightRequestCount() == 0)
                     timeout = Math.min(timeout, retryBackoffMs);
@@ -265,6 +285,8 @@ public void poll(long timeout, long now, PollCondition pollCondition, boolean di
 
             // clean unsent requests collection to keep the map from growing indefinitely
             unsent.clean();
+        } finally {
+            lock.unlock();
         }
 
         // called without the lock to avoid deadlock potential if handlers need to acquire locks
@@ -303,8 +325,11 @@ public boolean awaitPendingRequests(Node node, long timeoutMs) {
      * @return The number of pending requests
      */
     public int pendingRequestCount(Node node) {
-        synchronized (this) {
+        lock.lock();
+        try {
             return unsent.requestCount(node) + client.inFlightRequestCount(node.idString());
+        } finally {
+            lock.unlock();
         }
     }
 
@@ -317,8 +342,11 @@ public int pendingRequestCount(Node node) {
     public boolean hasPendingRequests(Node node) {
         if (unsent.hasRequests(node))
             return true;
-        synchronized (this) {
+        lock.lock();
+        try {
             return client.hasInFlightRequests(node.idString());
+        } finally {
+            lock.unlock();
         }
     }
 
@@ -328,8 +356,11 @@ public boolean hasPendingRequests(Node node) {
      * @return The total count of pending requests
      */
     public int pendingRequestCount() {
-        synchronized (this) {
+        lock.lock();
+        try {
             return unsent.requestCount() + client.inFlightRequestCount();
+        } finally {
+            lock.unlock();
         }
     }
 
@@ -341,8 +372,11 @@ public int pendingRequestCount() {
     public boolean hasPendingRequests() {
         if (unsent.hasRequests())
             return true;
-        synchronized (this) {
+        lock.lock();
+        try {
             return client.hasInFlightRequests();
+        } finally {
+            lock.unlock();
         }
     }
 
@@ -386,15 +420,25 @@ private void checkDisconnects(long now) {
         }
     }
 
-    public void disconnect(Node node) {
-        failUnsentRequests(node, DisconnectException.INSTANCE);
+    private void handlePendingDisconnects() {
+        lock.lock();
+        try {
+            while (true) {
+                Node node = pendingDisconnects.poll();
+                if (node == null)
+                    break;
 
-        synchronized (this) {
-            client.disconnect(node.idString());
+                failUnsentRequests(node, DisconnectException.INSTANCE);
+                client.disconnect(node.idString());
+            }
+        } finally {
+            lock.unlock();
         }
+    }
 
-        // We need to poll to ensure callbacks from in-flight requests on the disconnected socket are fired
-        pollNoWakeup();
+    public void disconnectAsync(Node node) {
+        pendingDisconnects.offer(node);
+        client.wakeup();
     }
 
     private void failExpiredRequests(long now) {
@@ -408,16 +452,16 @@ private void failExpiredRequests(long now) {
 
     private void failUnsentRequests(Node node, RuntimeException e) {
         // clear unsent requests to node and fail their corresponding futures
-        synchronized (this) {
+        lock.lock();
+        try {
             Collection unsentRequests = unsent.remove(node);
             for (ClientRequest unsentRequest : unsentRequests) {
                 RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) unsentRequest.callback();
                 handler.onFailure(e);
             }
+        } finally {
+            lock.unlock();
         }
-
-        // called without the lock to avoid deadlock potential
-        firePendingCompletedRequests();
     }
 
     private boolean trySend(long now) {
@@ -458,8 +502,11 @@ public void disableWakeups() {
 
     @Override
     public void close() throws IOException {
-        synchronized (this) {
+        lock.lock();
+        try {
             client.close();
+        } finally {
+            lock.unlock();
         }
     }
 
@@ -469,8 +516,11 @@ public void close() throws IOException {
      * @param node Node to connect to if possible
      */
     public boolean connectionFailed(Node node) {
-        synchronized (this) {
+        lock.lock();
+        try {
             return client.connectionFailed(node);
+        } finally {
+            lock.unlock();
         }
     }
 
@@ -481,8 +531,11 @@ public boolean connectionFailed(Node node) {
      * @param node The node to connect to
      */
     public void tryConnect(Node node) {
-        synchronized (this) {
+        lock.lock();
+        try {
             client.ready(node, time.milliseconds());
+        } finally {
+            lock.unlock();
         }
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java
index ed45e3af56d5b..faf9240537c35 100644
--- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.errors.AuthenticationException;
+import org.apache.kafka.common.errors.InterruptException;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.AbstractRequest;
@@ -86,6 +87,7 @@ public FutureResponse(Node node,
     private final Queue futureResponses = new ArrayDeque<>();
     private final Queue metadataUpdates = new ArrayDeque<>();
     private volatile NodeApiVersions nodeApiVersions = NodeApiVersions.create();
+    private volatile int numBlockingWakeups = 0;
 
     public MockClient(Time time) {
         this(time, null);
@@ -195,8 +197,40 @@ public void send(ClientRequest request, long now) {
         this.requests.add(request);
     }
 
+    /**
+     * Simulate a blocking poll in order to test wakeup behavior.
+     *
+     * @param numBlockingWakeups The number of polls which will block until woken up
+     */
+    public synchronized void enableBlockingUntilWakeup(int numBlockingWakeups) {
+        this.numBlockingWakeups = numBlockingWakeups;
+    }
+
+    @Override
+    public synchronized void wakeup() {
+        if (numBlockingWakeups > 0) {
+            numBlockingWakeups--;
+            notify();
+        }
+    }
+
+    private synchronized void maybeAwaitWakeup() {
+        try {
+            int remainingBlockingWakeups = numBlockingWakeups;
+            if (remainingBlockingWakeups <= 0)
+                return;
+
+            while (numBlockingWakeups == remainingBlockingWakeups)
+                wait();
+        } catch (InterruptedException e) {
+            throw new InterruptException(e);
+        }
+    }
+
     @Override
     public List poll(long timeoutMs, long now) {
+        maybeAwaitWakeup();
+
         List copy = new ArrayList<>(this.responses);
 
         if (metadata != null && metadata.updateRequested()) {
@@ -427,10 +461,6 @@ public ClientRequest newClientRequest(String nodeId, AbstractRequest.Builder
                 expectResponse, callback);
     }
 
-    @Override
-    public void wakeup() {
-    }
-
     @Override
     public void close() {
     }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 2e5c96019b903..ac392055f9b06 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -221,7 +221,8 @@ public void onComplete(Map offsets, Exception
             });
         }
 
-        coordinator.coordinatorDead();
+        coordinator.markCoordinatorUnknown();
+        consumerClient.pollNoWakeup();
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertEquals(numRequests, responses.get());
     }
@@ -238,7 +239,7 @@ public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() throws
         final AtomicBoolean asyncCallbackInvoked = new AtomicBoolean(false);
         Map offsets = Collections.singletonMap(
                 new TopicPartition("foo", 0), new OffsetCommitRequest.PartitionData(13L, ""));
-        consumerClient.send(coordinator.coordinator(), new OffsetCommitRequest.Builder(groupId, offsets))
+        consumerClient.send(coordinator.checkAndGetCoordinator(), new OffsetCommitRequest.Builder(groupId, offsets))
                 .compose(new RequestFutureAdapter() {
                     @Override
                     public void onSuccess(ClientResponse value, RequestFuture future) {}
@@ -251,7 +252,8 @@ public void onFailure(RuntimeException e, RequestFuture future) {
                     }
                 });
 
-        coordinator.coordinatorDead();
+        coordinator.markCoordinatorUnknown();
+        consumerClient.pollNoWakeup();
         assertTrue(asyncCallbackInvoked.get());
     }
 
@@ -1033,6 +1035,7 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error)
 
         client.respond(offsetCommitResponse(Collections.singletonMap(t1p, error)));
         consumerClient.pollNoWakeup();
+        consumerClient.pollNoWakeup(); // second poll since coordinator disconnect is async
         coordinator.invokeCompletedOffsetCommitCallbacks();
 
         assertTrue(coordinator.coordinatorUnknown());
@@ -1678,7 +1681,7 @@ public void testAutoCommitAfterCoordinatorBackToService() {
         subscriptions.assignFromUser(Collections.singleton(t1p));
         subscriptions.seek(t1p, 100L);
 
-        coordinator.coordinatorDead();
+        coordinator.markCoordinatorUnknown();
         assertTrue(coordinator.coordinatorUnknown());
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
index 904270ec48971..d0888fa56554b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
@@ -107,7 +107,8 @@ public void testDisconnectWithUnsentRequests() {
         RequestFuture future = consumerClient.send(node, heartbeat());
         assertTrue(consumerClient.hasPendingRequests(node));
         assertFalse(client.hasInFlightRequests(node.idString()));
-        consumerClient.disconnect(node);
+        consumerClient.disconnectAsync(node);
+        consumerClient.pollNoWakeup();
         assertTrue(future.failed());
         assertTrue(future.exception() instanceof DisconnectException);
     }
@@ -118,7 +119,8 @@ public void testDisconnectWithInFlightRequests() {
         consumerClient.pollNoWakeup();
         assertTrue(consumerClient.hasPendingRequests(node));
         assertTrue(client.hasInFlightRequests(node.idString()));
-        consumerClient.disconnect(node);
+        consumerClient.disconnectAsync(node);
+        consumerClient.pollNoWakeup();
         assertTrue(future.failed());
         assertTrue(future.exception() instanceof DisconnectException);
     }
@@ -205,6 +207,66 @@ public void wakeup() {
         assertTrue(future.isDone());
     }
 
+    @Test
+    public void testDisconnectWakesUpPoll() throws Exception {
+        final RequestFuture future = consumerClient.send(node, heartbeat());
+
+        client.enableBlockingUntilWakeup(1);
+        Thread t = new Thread() {
+            @Override
+            public void run() {
+                consumerClient.poll(future);
+            }
+        };
+        t.start();
+
+        consumerClient.disconnectAsync(node);
+        t.join();
+        assertTrue(future.failed());
+        assertTrue(future.exception() instanceof DisconnectException);
+    }
+
+    @Test
+    public void testFutureCompletionOutsidePoll() throws Exception {
+        // Tests the scenario in which the request that is being awaited in one thread
+        // is received and completed in another thread.
+
+        final RequestFuture future = consumerClient.send(node, heartbeat());
+        consumerClient.pollNoWakeup(); // dequeue and send the request
+
+        client.enableBlockingUntilWakeup(2);
+        Thread t1 = new Thread() {
+            @Override
+            public void run() {
+                consumerClient.pollNoWakeup();
+            }
+        };
+        t1.start();
+
+        // Sleep a little so that t1 is blocking in poll
+        Thread.sleep(50);
+
+        Thread t2 = new Thread() {
+            @Override
+            public void run() {
+                consumerClient.poll(future);
+            }
+        };
+        t2.start();
+
+        // Sleep a little so that t2 is awaiting the network client lock
+        Thread.sleep(50);
+
+        // Simulate a network response and return from the poll in t1
+        client.respond(heartbeatResponse(Errors.NONE));
+        client.wakeup();
+
+        // Both threads should complete since t1 should wakeup t2
+        t1.join();
+        t2.join();
+        assertTrue(future.succeeded());
+    }
+
     @Test
     public void testAwaitForMetadataUpdateWithTimeout() {
         assertFalse(consumerClient.awaitMetadataUpdate(10L));

From 9868976747fb4a7a4a9b18e1f4050633c226be7d Mon Sep 17 00:00:00 2001
From: Sandor Murakozi 
Date: Thu, 1 Mar 2018 18:12:52 -0800
Subject: [PATCH 0110/1847] KAFKA-6111: Improve test coverage of KafkaZkClient,
 fix bugs found by new tests;

---
 .../main/scala/kafka/zk/KafkaZkClient.scala   |   5 +-
 .../unit/kafka/zk/KafkaZkClientTest.scala     | 576 +++++++++++++++++-
 2 files changed, 555 insertions(+), 26 deletions(-)

diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index 6545fde30e947..d61b281eed1ae 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -88,7 +88,8 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   def updateBrokerInfoInZk(brokerInfo: BrokerInfo): Unit = {
     val brokerIdPath = brokerInfo.path
     val setDataRequest = SetDataRequest(brokerIdPath, brokerInfo.toJsonBytes, ZkVersion.NoVersion)
-    retryRequestUntilConnected(setDataRequest)
+    val response = retryRequestUntilConnected(setDataRequest)
+    response.maybeThrow()
     info("Updated broker %d at path %s with addresses: %s".format(brokerInfo.broker.id, brokerIdPath, brokerInfo.broker.endPoints))
   }
 
@@ -424,7 +425,7 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   def deleteLogDirEventNotifications(): Unit = {
     val getChildrenResponse = retryRequestUntilConnected(GetChildrenRequest(LogDirEventNotificationZNode.path))
     if (getChildrenResponse.resultCode == Code.OK) {
-      deleteLogDirEventNotifications(getChildrenResponse.children)
+      deleteLogDirEventNotifications(getChildrenResponse.children.map(LogDirEventNotificationSequenceZNode.sequenceNumber))
     } else if (getChildrenResponse.resultCode != Code.NONODE) {
       getChildrenResponse.maybeThrow
     }
diff --git a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
index d3726c25c5899..e44c2c94e52ed 100644
--- a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
@@ -16,10 +16,11 @@
 */
 package kafka.zk
 
-import java.util.{Properties, UUID}
+import java.util.{Collections, Properties, UUID}
 import java.nio.charset.StandardCharsets.UTF_8
+import java.util.concurrent.{CountDownLatch, TimeUnit}
 
-import kafka.api.ApiVersion
+import kafka.api.{ApiVersion, LeaderAndIsr}
 import kafka.cluster.{Broker, EndPoint}
 import kafka.log.LogConfig
 import kafka.security.auth._
@@ -29,17 +30,48 @@ import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
 import org.apache.kafka.common.security.token.delegation.TokenInformation
-import org.apache.kafka.common.utils.SecurityUtils
-import org.apache.zookeeper.KeeperException.NodeExistsException
+import org.apache.kafka.common.utils.{SecurityUtils, Time}
+import org.apache.zookeeper.KeeperException.{Code, NoNodeException, NodeExistsException}
 import org.junit.Assert._
-import org.junit.Test
-
+import org.junit.{After, Before, Test}
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.{Seq, mutable}
 import scala.util.Random
 
+import kafka.controller.LeaderIsrAndControllerEpoch
+import kafka.zk.KafkaZkClient.UpdateLeaderAndIsrResult
+import kafka.zookeeper._
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.zookeeper.data.Stat
+
 class KafkaZkClientTest extends ZooKeeperTestHarness {
 
   private val group = "my-group"
+  private val topic1 = "topic1"
+  private val topic2 = "topic2"
+
+  val topicPartition10 = new TopicPartition(topic1, 0)
+  val topicPartition11 = new TopicPartition(topic1, 1)
+  val topicPartition20 = new TopicPartition(topic2, 0)
+  val topicPartitions10_11 = Seq(topicPartition10, topicPartition11)
+
+  var otherZkClient: KafkaZkClient = _
+
+  @Before
+  override def setUp(): Unit = {
+    super.setUp()
+    otherZkClient = KafkaZkClient(zkConnect, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled), zkSessionTimeout,
+      zkConnectionTimeout, zkMaxInFlightRequests, Time.SYSTEM)
+  }
+
+  @After
+  override def tearDown(): Unit = {
+    if (otherZkClient != null)
+      otherZkClient.close()
+    super.tearDown()
+  }
+
   private val topicPartition = new TopicPartition("topic", 0)
 
   @Test
@@ -90,10 +122,10 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
   @Test
   def testTopicAssignmentMethods() {
-    val topic1 = "topic1"
-    val topic2 = "topic2"
+    assertTrue(zkClient.getAllTopicsInCluster.isEmpty)
 
     // test with non-existing topic
+    assertFalse(zkClient.topicExists(topic1))
     assertTrue(zkClient.getTopicPartitionCount(topic1).isEmpty)
     assertTrue(zkClient.getPartitionAssignmentForTopics(Set(topic1)).isEmpty)
     assertTrue(zkClient.getPartitionsForTopics(Set(topic1)).isEmpty)
@@ -108,6 +140,8 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     // create a topic assignment
     zkClient.createTopicAssignment(topic1, assignment)
 
+    assertTrue(zkClient.topicExists(topic1))
+
     val expectedAssignment = assignment map { topicAssignment =>
       val partition = topicAssignment._1.partition
       val assignment = topicAssignment._2
@@ -214,6 +248,43 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     assertEquals(Some("""{"version":1,"partitions":[{"topic":"topic-b","partition":0}]}"""), dataAsString(expectedPath))
   }
 
+  @Test
+  def testIsrChangeNotificationGetters(): Unit = {
+    assertEquals("Failed for non existing parent ZK node", Seq.empty, zkClient.getAllIsrChangeNotifications)
+    assertEquals("Failed for non existing parent ZK node", Seq.empty, zkClient.getPartitionsFromIsrChangeNotifications(Seq("0000000000")))
+
+    zkClient.createRecursive("/isr_change_notification")
+
+    zkClient.propagateIsrChanges(Set(topicPartition10, topicPartition11))
+    zkClient.propagateIsrChanges(Set(topicPartition10))
+
+    assertEquals(Set("0000000000", "0000000001"), zkClient.getAllIsrChangeNotifications.toSet)
+
+    // A partition can have multiple notifications
+    assertEquals(Seq(topicPartition10, topicPartition11, topicPartition10),
+      zkClient.getPartitionsFromIsrChangeNotifications(Seq("0000000000", "0000000001")))
+  }
+
+  @Test
+  def testIsrChangeNotificationsDeletion(): Unit = {
+    // Should not fail even if parent node does not exist
+    zkClient.deleteIsrChangeNotifications(Seq("0000000000"))
+
+    zkClient.createRecursive("/isr_change_notification")
+
+    zkClient.propagateIsrChanges(Set(topicPartition10, topicPartition11))
+    zkClient.propagateIsrChanges(Set(topicPartition10))
+    zkClient.propagateIsrChanges(Set(topicPartition11))
+
+    zkClient.deleteIsrChangeNotifications(Seq("0000000001"))
+    // Should not fail if called on a non-existent notification
+    zkClient.deleteIsrChangeNotifications(Seq("0000000001"))
+
+    assertEquals(Set("0000000000", "0000000002"), zkClient.getAllIsrChangeNotifications.toSet)
+    zkClient.deleteIsrChangeNotifications()
+    assertEquals(Seq.empty,zkClient.getAllIsrChangeNotifications)
+  }
+
   @Test
   def testPropagateLogDir(): Unit = {
     zkClient.createRecursive("/log_dir_event_notification")
@@ -237,6 +308,54 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     assertEquals(Some("""{"version":1,"broker":4,"event":1}"""), dataAsString(expectedPath))
   }
 
+  @Test
+  def testLogDirGetters(): Unit = {
+    assertEquals("getAllLogDirEventNotifications failed for non existing parent ZK node",
+      Seq.empty, zkClient.getAllLogDirEventNotifications)
+    assertEquals("getBrokerIdsFromLogDirEvents failed for non existing parent ZK node",
+      Seq.empty, zkClient.getBrokerIdsFromLogDirEvents(Seq("0000000000")))
+
+    zkClient.createRecursive("/log_dir_event_notification")
+
+    val brokerId = 3
+    zkClient.propagateLogDirEvent(brokerId)
+
+    assertEquals(Seq(3), zkClient.getBrokerIdsFromLogDirEvents(Seq("0000000000")))
+
+    zkClient.propagateLogDirEvent(brokerId)
+
+    val anotherBrokerId = 4
+    zkClient.propagateLogDirEvent(anotherBrokerId)
+
+    val notifications012 = Seq("0000000000", "0000000001", "0000000002")
+    assertEquals(notifications012.toSet, zkClient.getAllLogDirEventNotifications.toSet)
+    assertEquals(Seq(3, 3, 4), zkClient.getBrokerIdsFromLogDirEvents(notifications012))
+  }
+
+  @Test
+  def testLogDirEventNotificationsDeletion(): Unit = {
+    // Should not fail even if parent node does not exist
+    zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"))
+
+    zkClient.createRecursive("/log_dir_event_notification")
+
+    val brokerId = 3
+    val anotherBrokerId = 4
+
+    zkClient.propagateLogDirEvent(brokerId)
+    zkClient.propagateLogDirEvent(brokerId)
+    zkClient.propagateLogDirEvent(anotherBrokerId)
+
+    zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"))
+
+    assertEquals(Seq("0000000001"), zkClient.getAllLogDirEventNotifications)
+
+    zkClient.propagateLogDirEvent(anotherBrokerId)
+
+    zkClient.deleteLogDirEventNotifications()
+    assertEquals(Seq.empty, zkClient.getAllLogDirEventNotifications)
+  }
+
   @Test
   def testSetGetAndDeletePartitionReassignment() {
     zkClient.createRecursive(AdminZNode.path)
@@ -377,10 +496,23 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testDeleteTopicPathMethods() {
-    val topic1 = "topic1"
-    val topic2 = "topic2"
+  def testDeletePath(): Unit = {
+    val path = "/a/b/c"
+    zkClient.createRecursive(path)
+    zkClient.deletePath(path)
+    assertFalse(zkClient.pathExists(path))
+  }
+
+  @Test
+  def testDeleteTopicZNode(): Unit = {
+    zkClient.deleteTopicZNode(topic1)
+    zkClient.createRecursive(TopicZNode.path(topic1))
+    zkClient.deleteTopicZNode(topic1)
+    assertFalse(zkClient.pathExists(TopicZNode.path(topic1)))
+  }
 
+  @Test
+  def testDeleteTopicPathMethods() {
     assertFalse(zkClient.isTopicMarkedForDeletion(topic1))
     assertTrue(zkClient.getTopicDeletions.isEmpty)
 
@@ -394,18 +526,26 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     assertTrue(zkClient.getTopicDeletions.isEmpty)
   }
 
+  private def assertPathExistenceAndData(expectedPath: String, data: String): Unit = {
+    assertTrue(zkClient.pathExists(expectedPath))
+    assertEquals(Some(data), dataAsString(expectedPath))
+   }
+
+  @Test
+  def testCreateTokenChangeNotification(): Unit = {
+    intercept[NoNodeException] {
+      zkClient.createTokenChangeNotification("delegationToken")
+    }
+    zkClient.createDelegationTokenPaths()
+
+    zkClient.createTokenChangeNotification("delegationToken")
+    assertPathExistenceAndData("/delegation_token/token_changes/token_change_0000000000", "delegationToken")
+  }
+
   @Test
   def testEntityConfigManagementMethods() {
-    val topic1 = "topic1"
-    val topic2 = "topic2"
-
     assertTrue(zkClient.getEntityConfigs(ConfigType.Topic, topic1).isEmpty)
 
-    val logProps = new Properties()
-    logProps.put(LogConfig.SegmentBytesProp, "1024")
-    logProps.put(LogConfig.SegmentIndexBytesProp, "1024")
-    logProps.put(LogConfig.CleanupPolicyProp, LogConfig.Compact)
-
     zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic1, logProps)
     assertEquals(logProps, zkClient.getEntityConfigs(ConfigType.Topic, topic1))
 
@@ -421,15 +561,399 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   }
 
   @Test
-  def testBrokerRegistrationMethods() {
+  def testCreateConfigChangeNotification(): Unit = {
+    intercept[NoNodeException] {
+      zkClient.createConfigChangeNotification(ConfigEntityZNode.path(ConfigType.Topic, topic1))
+    }
+
+    zkClient.createTopLevelPaths()
+    zkClient.createConfigChangeNotification(ConfigEntityZNode.path(ConfigType.Topic, topic1))
+
+    assertPathExistenceAndData(
+      "/config/changes/config_change_0000000000",
+      """{"version":2,"entity_path":"/config/topics/topic1"}""")
+  }
+
+  private def createLogProps(bytesProp: Int): Properties = {
+    val logProps = new Properties()
+    logProps.put(LogConfig.SegmentBytesProp, bytesProp.toString)
+    logProps.put(LogConfig.SegmentIndexBytesProp, bytesProp.toString)
+    logProps.put(LogConfig.CleanupPolicyProp, LogConfig.Compact)
+    logProps
+  }
+
+  private val logProps = createLogProps(1024)
+
+  @Test
+  def testGetLogConfigs(): Unit = {
+    val emptyConfig = LogConfig(Collections.emptyMap())
+    assertEquals("Non existent config, no defaults",
+      (Map(topic1 -> emptyConfig), Map.empty),
+      zkClient.getLogConfigs(Seq(topic1), Collections.emptyMap()))
+
+    val logProps2 = createLogProps(2048)
+
+    zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic1, logProps)
+    assertEquals("One existing and one non-existent topic",
+      (Map(topic1 -> LogConfig(logProps), topic2 -> emptyConfig), Map.empty),
+      zkClient.getLogConfigs(Seq(topic1, topic2), Collections.emptyMap()))
+
+    zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic2, logProps2)
+    assertEquals("Two existing topics",
+      (Map(topic1 -> LogConfig(logProps), topic2 -> LogConfig(logProps2)), Map.empty),
+      zkClient.getLogConfigs(Seq(topic1, topic2), Collections.emptyMap()))
+
+    val logProps1WithMoreValues = createLogProps(1024)
+    logProps1WithMoreValues.put(LogConfig.SegmentJitterMsProp, "100")
+    logProps1WithMoreValues.put(LogConfig.SegmentBytesProp, "1024")
+
+    assertEquals("Config with defaults",
+      (Map(topic1 -> LogConfig(logProps1WithMoreValues)), Map.empty),
+      zkClient.getLogConfigs(Seq(topic1),
+        Map[String, AnyRef](LogConfig.SegmentJitterMsProp -> "100", LogConfig.SegmentBytesProp -> "128").asJava))
+  }
+
+  private def createBrokerInfo(id: Int, host: String, port: Int, securityProtocol: SecurityProtocol,
+                               rack: Option[String] = None): BrokerInfo =
+    BrokerInfo(Broker(id, Seq(new EndPoint(host, port, ListenerName.forSecurityProtocol
+    (securityProtocol), securityProtocol)), rack = rack), ApiVersion.latestVersion, jmxPort = port + 10)
+
+  @Test
+  def testRegisterBrokerInfo(): Unit = {
     zkClient.createTopLevelPaths()
 
-    val brokerInfo = BrokerInfo(Broker(1,
-      Seq(new EndPoint("test.host", 9999, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), SecurityProtocol.PLAINTEXT)),
-      rack = None), ApiVersion.latestVersion, jmxPort = 9998)
+    val brokerInfo = createBrokerInfo(1, "test.host", 9999, SecurityProtocol.PLAINTEXT)
+    val differentBrokerInfoWithSameId = createBrokerInfo(1, "test.host2", 9995, SecurityProtocol.SSL)
 
     zkClient.registerBrokerInZk(brokerInfo)
     assertEquals(Some(brokerInfo.broker), zkClient.getBroker(1))
+    assertEquals("Other ZK clients can read broker info", Some(brokerInfo.broker), otherZkClient.getBroker(1))
+
+    // Node exists, owned by current session - no error, no update
+    zkClient.registerBrokerInZk(differentBrokerInfoWithSameId)
+    assertEquals(Some(brokerInfo.broker), zkClient.getBroker(1))
+
+    // Other client tries to register broker with same id causes failure, info is not changed in ZK
+    intercept[NodeExistsException] {
+      otherZkClient.registerBrokerInZk(differentBrokerInfoWithSameId)
+    }
+    assertEquals(Some(brokerInfo.broker), zkClient.getBroker(1))
+  }
+
+  @Test
+  def testGetBrokerMethods(): Unit = {
+    zkClient.createTopLevelPaths()
+
+    assertEquals(Seq.empty,zkClient.getAllBrokersInCluster)
+    assertEquals(Seq.empty, zkClient.getSortedBrokerList())
+    assertEquals(None, zkClient.getBroker(0))
+
+    val brokerInfo0 = createBrokerInfo(0, "test.host0", 9998, SecurityProtocol.PLAINTEXT)
+    val brokerInfo1 = createBrokerInfo(1, "test.host1", 9999, SecurityProtocol.SSL)
+
+    zkClient.registerBrokerInZk(brokerInfo1)
+    otherZkClient.registerBrokerInZk(brokerInfo0)
+
+    assertEquals(Seq(0, 1), zkClient.getSortedBrokerList())
+    assertEquals(
+      Seq(brokerInfo0.broker, brokerInfo1.broker),
+      zkClient.getAllBrokersInCluster
+    )
+    assertEquals(Some(brokerInfo0.broker), zkClient.getBroker(0))
+  }
+
+  @Test
+  def testUpdateBrokerInfo(): Unit = {
+    zkClient.createTopLevelPaths()
+
+    // Updating info of a broker not existing in ZK fails
+    val originalBrokerInfo = createBrokerInfo(1, "test.host", 9999, SecurityProtocol.PLAINTEXT)
+    intercept[NoNodeException]{
+      zkClient.updateBrokerInfoInZk(originalBrokerInfo)
+    }
+
+    zkClient.registerBrokerInZk(originalBrokerInfo)
+
+    val updatedBrokerInfo = createBrokerInfo(1, "test.host2", 9995, SecurityProtocol.SSL)
+    zkClient.updateBrokerInfoInZk(updatedBrokerInfo)
+    assertEquals(Some(updatedBrokerInfo.broker), zkClient.getBroker(1))
+
+    // Other ZK clients can update info
+    otherZkClient.updateBrokerInfoInZk(originalBrokerInfo)
+    assertEquals(Some(originalBrokerInfo.broker), otherZkClient.getBroker(1))
+  }
+
+  private def statWithVersion(version: Int): Stat = {
+    val stat = new Stat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+    stat.setVersion(version)
+    stat
+  }
+
+  private def leaderIsrAndControllerEpochs(state: Int, zkVersion: Int): Map[TopicPartition, LeaderIsrAndControllerEpoch] =
+    Map(
+      topicPartition10 -> LeaderIsrAndControllerEpoch(
+        LeaderAndIsr(leader = 1, leaderEpoch = state, isr = List(2 + state, 3 + state), zkVersion = zkVersion),
+        controllerEpoch = 4),
+      topicPartition11 -> LeaderIsrAndControllerEpoch(
+        LeaderAndIsr(leader = 0, leaderEpoch = state + 1, isr = List(1 + state, 2 + state), zkVersion = zkVersion),
+        controllerEpoch = 4))
+
+  val initialLeaderIsrAndControllerEpochs: Map[TopicPartition, LeaderIsrAndControllerEpoch] =
+    leaderIsrAndControllerEpochs(0, 0)
+
+  val initialLeaderIsrs: Map[TopicPartition, LeaderAndIsr] = initialLeaderIsrAndControllerEpochs.mapValues(_.leaderAndIsr)
+  private def leaderIsrs(state: Int, zkVersion: Int): Map[TopicPartition, LeaderAndIsr] =
+    leaderIsrAndControllerEpochs(state, zkVersion).mapValues(_.leaderAndIsr)
+
+  private def checkUpdateLeaderAndIsrResult(
+                  expectedSuccessfulPartitions: Map[TopicPartition, LeaderAndIsr],
+                  expectedPartitionsToRetry: Seq[TopicPartition],
+                  expectedFailedPartitions: Map[TopicPartition, (Class[_], String)],
+                  actualUpdateLeaderAndIsrResult: UpdateLeaderAndIsrResult): Unit = {
+    val failedPartitionsExcerpt =
+      actualUpdateLeaderAndIsrResult.failedPartitions.mapValues(e => (e.getClass, e.getMessage))
+    assertEquals("Permanently failed updates do not match expected",
+      expectedFailedPartitions, failedPartitionsExcerpt)
+    assertEquals("Retriable updates (due to BADVERSION) do not match expected",
+      expectedPartitionsToRetry, actualUpdateLeaderAndIsrResult.partitionsToRetry)
+    assertEquals("Successful updates do not match expected",
+      expectedSuccessfulPartitions, actualUpdateLeaderAndIsrResult.successfulPartitions)
+  }
+
+  @Test
+  def testUpdateLeaderAndIsr(): Unit = {
+    zkClient.createRecursive(TopicZNode.path(topic1))
+
+    // Non-existing topicPartitions
+    checkUpdateLeaderAndIsrResult(
+        Map.empty,
+        mutable.ArrayBuffer.empty,
+      Map(
+        topicPartition10 -> (classOf[NoNodeException], "KeeperErrorCode = NoNode for /brokers/topics/topic1/partitions/0/state"),
+        topicPartition11 -> (classOf[NoNodeException], "KeeperErrorCode = NoNode for /brokers/topics/topic1/partitions/1/state")),
+      zkClient.updateLeaderAndIsr(initialLeaderIsrs, controllerEpoch = 4))
+
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+
+    checkUpdateLeaderAndIsrResult(
+      leaderIsrs(state = 1, zkVersion = 1),
+      mutable.ArrayBuffer.empty,
+      Map.empty,
+      zkClient.updateLeaderAndIsr(leaderIsrs(state = 1, zkVersion = 0),controllerEpoch = 4))
+
+    // Try to update with wrong ZK version
+    checkUpdateLeaderAndIsrResult(
+      Map.empty,
+      ArrayBuffer(topicPartition10, topicPartition11),
+      Map.empty,
+      zkClient.updateLeaderAndIsr(leaderIsrs(state = 1, zkVersion = 0),controllerEpoch = 4))
+
+    // Trigger successful, to be retried and failed partitions in same call
+    val mixedState = Map(
+      topicPartition10 -> LeaderAndIsr(leader = 1, leaderEpoch = 2, isr = List(4, 5), zkVersion = 1),
+      topicPartition11 -> LeaderAndIsr(leader = 0, leaderEpoch = 2, isr = List(3, 4), zkVersion = 0),
+      topicPartition20 -> LeaderAndIsr(leader = 0, leaderEpoch = 2, isr = List(3, 4), zkVersion = 0))
+
+    checkUpdateLeaderAndIsrResult(
+      leaderIsrs(state = 2, zkVersion = 2).filterKeys{_ == topicPartition10},
+      ArrayBuffer(topicPartition11),
+      Map(
+        topicPartition20 -> (classOf[NoNodeException], "KeeperErrorCode = NoNode for /brokers/topics/topic2/partitions/0/state")),
+      zkClient.updateLeaderAndIsr(mixedState, controllerEpoch = 4))
+  }
+
+  private def checkGetDataResponse(
+      leaderIsrAndControllerEpochs: Map[TopicPartition,LeaderIsrAndControllerEpoch],
+      topicPartition: TopicPartition,
+      response: GetDataResponse): Unit = {
+    val zkVersion = leaderIsrAndControllerEpochs(topicPartition).leaderAndIsr.zkVersion
+    assertEquals(Code.OK, response.resultCode)
+    assertEquals(TopicPartitionStateZNode.path(topicPartition), response.path)
+    assertEquals(Some(topicPartition), response.ctx)
+    assertEquals(
+      Some(leaderIsrAndControllerEpochs(topicPartition)),
+      TopicPartitionStateZNode.decode(response.data, statWithVersion(zkVersion)))
+  }
+
+  private def eraseMetadata(response: CreateResponse): CreateResponse =
+    response.copy(metadata = ResponseMetadata(0, 0))
+
+  @Test
+  def testGetTopicsAndPartitions(): Unit = {
+    assertTrue(zkClient.getAllTopicsInCluster.isEmpty)
+    assertTrue(zkClient.getAllPartitions.isEmpty)
+
+    zkClient.createRecursive(TopicZNode.path(topic1))
+    zkClient.createRecursive(TopicZNode.path(topic2))
+    assertEquals(Set(topic1, topic2), zkClient.getAllTopicsInCluster.toSet)
+
+    assertTrue(zkClient.getAllPartitions.isEmpty)
+
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+    assertEquals(Set(topicPartition10, topicPartition11), zkClient.getAllPartitions)
+  }
+
+  @Test
+  def testCreateAndGetTopicPartitionStatesRaw(): Unit = {
+    zkClient.createRecursive(TopicZNode.path(topic1))
+
+    assertEquals(
+      Seq(
+        CreateResponse(Code.OK, TopicPartitionStateZNode.path(topicPartition10), Some(topicPartition10),
+          TopicPartitionStateZNode.path(topicPartition10), ResponseMetadata(0, 0)),
+        CreateResponse(Code.OK, TopicPartitionStateZNode.path(topicPartition11), Some(topicPartition11),
+          TopicPartitionStateZNode.path(topicPartition11), ResponseMetadata(0, 0))),
+      zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+        .map(eraseMetadata).toList)
+
+    val getResponses = zkClient.getTopicPartitionStatesRaw(topicPartitions10_11)
+    assertEquals(2, getResponses.size)
+    topicPartitions10_11.zip(getResponses) foreach {case (tp, r) => checkGetDataResponse(initialLeaderIsrAndControllerEpochs, tp, r)}
+
+    // Trying to create existing topicPartition states fails
+    assertEquals(
+      Seq(
+        CreateResponse(Code.NODEEXISTS, TopicPartitionStateZNode.path(topicPartition10), Some(topicPartition10),
+          null, ResponseMetadata(0, 0)),
+        CreateResponse(Code.NODEEXISTS, TopicPartitionStateZNode.path(topicPartition11), Some(topicPartition11),
+          null, ResponseMetadata(0, 0))),
+      zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs).map(eraseMetadata).toList)
+  }
+
+  @Test
+  def testSetTopicPartitionStatesRaw(): Unit = {
+
+    def expectedSetDataResponses(topicPartitions: TopicPartition*)(resultCode: Code, stat: Stat) =
+      topicPartitions.map { topicPartition =>
+        SetDataResponse(resultCode, TopicPartitionStateZNode.path(topicPartition),
+          Some(topicPartition), stat, ResponseMetadata(0, 0))
+      }
+
+    zkClient.createRecursive(TopicZNode.path(topic1))
+
+    // Trying to set non-existing topicPartition's data results in NONODE responses
+    assertEquals(
+      expectedSetDataResponses(topicPartition10, topicPartition11)(Code.NONODE, null),
+      zkClient.setTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs).map {
+        _.copy(metadata = ResponseMetadata(0, 0))}.toList)
+
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+
+    assertEquals(
+      expectedSetDataResponses(topicPartition10, topicPartition11)(Code.OK, statWithVersion(1)),
+      zkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 1, zkVersion = 0)).map {
+        eraseMetadataAndStat}.toList)
+
+
+    val getResponses = zkClient.getTopicPartitionStatesRaw(topicPartitions10_11)
+    assertEquals(2, getResponses.size)
+    topicPartitions10_11.zip(getResponses) foreach {case (tp, r) => checkGetDataResponse(leaderIsrAndControllerEpochs(state = 1, zkVersion = 0), tp, r)}
+
+    // Other ZK client can also write the state of a partition
+    assertEquals(
+      expectedSetDataResponses(topicPartition10, topicPartition11)(Code.OK, statWithVersion(2)),
+      otherZkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 2, zkVersion = 1)).map {
+        eraseMetadataAndStat}.toList)
+  }
+
+  @Test
+  def testReassignPartitionsInProgress(): Unit = {
+    assertFalse(zkClient.reassignPartitionsInProgress)
+    zkClient.createRecursive(ReassignPartitionsZNode.path)
+    assertTrue(zkClient.reassignPartitionsInProgress)
+  }
+
+  @Test
+  def testGetTopicPartitionStates(): Unit = {
+    assertEquals(None, zkClient.getTopicPartitionState(topicPartition10))
+    assertEquals(None, zkClient.getLeaderForPartition(topicPartition10))
+
+    zkClient.createRecursive(TopicZNode.path(topic1))
+
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+    assertEquals(
+      initialLeaderIsrAndControllerEpochs,
+      zkClient.getTopicPartitionStates(Seq(topicPartition10, topicPartition11))
+    )
+
+    assertEquals(
+      Some(initialLeaderIsrAndControllerEpochs(topicPartition10)),
+      zkClient.getTopicPartitionState(topicPartition10)
+    )
+
+    assertEquals(Some(1), zkClient.getLeaderForPartition(topicPartition10))
+
+    val notExistingPartition = new TopicPartition(topic1, 2)
+    assertTrue(zkClient.getTopicPartitionStates(Seq(notExistingPartition)).isEmpty)
+    assertEquals(
+      Map(topicPartition10 -> initialLeaderIsrAndControllerEpochs(topicPartition10)),
+      zkClient.getTopicPartitionStates(Seq(topicPartition10, notExistingPartition))
+    )
+
+    assertEquals(None, zkClient.getTopicPartitionState(notExistingPartition))
+    assertEquals(None, zkClient.getLeaderForPartition(notExistingPartition))
+
+  }
+
+  private def eraseMetadataAndStat(response: SetDataResponse): SetDataResponse = {
+    val stat = if (response.stat != null) statWithVersion(response.stat.getVersion) else null
+    response.copy(metadata = ResponseMetadata(0, 0), stat = stat)
+  }
+
+  @Test
+  def testControllerEpochMethods(): Unit = {
+    assertEquals(None, zkClient.getControllerEpoch)
+
+    assertEquals("Setting non existing nodes should return NONODE results",
+      SetDataResponse(Code.NONODE, ControllerEpochZNode.path, None, null, ResponseMetadata(0, 0)),
+      eraseMetadataAndStat(zkClient.setControllerEpochRaw(1, 0)))
+
+    assertEquals("Creating non existing nodes is OK",
+      CreateResponse(Code.OK, ControllerEpochZNode.path, None, ControllerEpochZNode.path, ResponseMetadata(0, 0)),
+      eraseMetadata(zkClient.createControllerEpochRaw(0)))
+    assertEquals(0, zkClient.getControllerEpoch.get._1)
+
+    assertEquals("Attemt to create existing nodes should return NODEEXISTS",
+      CreateResponse(Code.NODEEXISTS, ControllerEpochZNode.path, None, null, ResponseMetadata(0, 0)),
+      eraseMetadata(zkClient.createControllerEpochRaw(0)))
+
+    assertEquals("Updating existing nodes is OK",
+      SetDataResponse(Code.OK, ControllerEpochZNode.path, None, statWithVersion(1), ResponseMetadata(0, 0)),
+      eraseMetadataAndStat(zkClient.setControllerEpochRaw(1, 0)))
+    assertEquals(1, zkClient.getControllerEpoch.get._1)
+
+    assertEquals("Updating with wrong ZK version returns BADVERSION",
+      SetDataResponse(Code.BADVERSION, ControllerEpochZNode.path, None, null, ResponseMetadata(0, 0)),
+      eraseMetadataAndStat(zkClient.setControllerEpochRaw(1, 0)))
+  }
+
+  @Test
+  def testControllerManagementMethods(): Unit = {
+    // No controller
+    assertEquals(None, zkClient.getControllerId)
+    // Create controller
+    zkClient.checkedEphemeralCreate(ControllerZNode.path, ControllerZNode.encode(brokerId = 1, timestamp = 123456))
+    assertEquals(Some(1), zkClient.getControllerId)
+    zkClient.deleteController()
+    assertEquals(None, zkClient.getControllerId)
+  }
+
+  @Test
+  def testZNodeChangeHandlerForDataChange(): Unit = {
+    val mockPath = "/foo"
+
+    val znodeChangeHandlerCountDownLatch = new CountDownLatch(1)
+    val zNodeChangeHandler = new ZNodeChangeHandler {
+      override def handleCreation(): Unit = {
+        znodeChangeHandlerCountDownLatch.countDown()
+      }
+
+      override val path: String = mockPath
+    }
+
+    zkClient.registerZNodeChangeHandlerAndCheckExistence(zNodeChangeHandler)
+    zkClient.createRecursive(mockPath)
+    assertTrue("Failed to receive create notification", znodeChangeHandlerCountDownLatch.await(5, TimeUnit.SECONDS))
   }
 
   @Test
@@ -458,7 +982,6 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     assertTrue(zkClient.getPreferredReplicaElection.isEmpty)
 
-    val topic1 = "topic1"
     val electionPartitions = Set(new TopicPartition(topic1, 0), new TopicPartition(topic1, 1))
 
     zkClient.createPreferredReplicaElection(electionPartitions)
@@ -498,6 +1021,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     // test non-existent token
     assertTrue(zkClient.getDelegationTokenInfo(tokenId).isEmpty)
+    assertFalse(zkClient.deleteDelegationToken(tokenId))
 
     // create a token
     zkClient.setOrCreateDelegationToken(token)
@@ -511,5 +1035,9 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     //test updated token
     assertEquals(tokenInfo, zkClient.getDelegationTokenInfo(tokenId).get)
+
+    //test deleting token
+    assertTrue(zkClient.deleteDelegationToken(tokenId))
+    assertEquals(None, zkClient.getDelegationTokenInfo(tokenId))
   }
 }

From 604b93cfdee7a8c5c879ef2217d50be88e39ac89 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Sat, 3 Mar 2018 13:27:29 -0800
Subject: [PATCH 0111/1847] =?UTF-8?q?KAFKA-6606;=20Ensure=20consumer=20awa?=
 =?UTF-8?q?its=20auto-commit=20interval=20after=20sending=E2=80=A6=20(#464?=
 =?UTF-8?q?1)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We need to reset the auto-commit deadline after sending the offset commit request so that we do not resend it while the request is still inflight.

Added unit tests ensuring this behavior and proper backoff in the case of a failure.

Reviewers: Guozhang Wang 
---
 .../internals/ConsumerCoordinator.java        |  12 +-
 .../internals/ConsumerCoordinatorTest.java    | 307 ++++++++++++------
 2 files changed, 210 insertions(+), 109 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 2afa1ff923685..3c99c966d54d8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -621,6 +621,7 @@ public boolean commitOffsetsSync(Map offsets,
 
     public void maybeAutoCommitOffsetsAsync(long now) {
         if (autoCommitEnabled && now >= nextAutoCommitDeadline) {
+            this.nextAutoCommitDeadline = now + autoCommitIntervalMs;
             doAutoCommitOffsetsAsync();
         }
     }
@@ -633,14 +634,15 @@ private void doAutoCommitOffsetsAsync() {
             @Override
             public void onComplete(Map offsets, Exception exception) {
                 if (exception != null) {
-                    log.warn("Asynchronous auto-commit of offsets {} failed: {}", offsets, exception.getMessage());
-                    if (exception instanceof RetriableException)
+                    if (exception instanceof RetriableException) {
+                        log.debug("Asynchronous auto-commit of offsets {} failed due to retriable error: {}", offsets,
+                                exception);
                         nextAutoCommitDeadline = Math.min(time.milliseconds() + retryBackoffMs, nextAutoCommitDeadline);
-                    else
-                        nextAutoCommitDeadline = time.milliseconds() + autoCommitIntervalMs;
+                    } else {
+                        log.warn("Asynchronous auto-commit of offsets {} failed: {}", offsets, exception.getMessage());
+                    }
                 } else {
                     log.debug("Completed asynchronous auto-commit of offsets {}", offsets);
-                    nextAutoCommitDeadline = time.milliseconds() + autoCommitIntervalMs;
                 }
             }
         });
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index ac392055f9b06..3e3c423a428b2 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -99,7 +99,6 @@ public class ConsumerCoordinatorTest {
     private int sessionTimeoutMs = 10000;
     private int heartbeatIntervalMs = 5000;
     private long retryBackoffMs = 100;
-    private boolean autoCommitEnabled = false;
     private int autoCommitIntervalMs = 2000;
     private MockPartitionAssignor partitionAssignor = new MockPartitionAssignor();
     private List assignors = Collections.singletonList(partitionAssignor);
@@ -134,7 +133,7 @@ public void setup() {
         this.partitionAssignor.clear();
 
         client.setNode(node);
-        this.coordinator = buildCoordinator(metrics, assignors, ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, autoCommitEnabled, true);
+        this.coordinator = buildCoordinator(metrics, assignors, ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, false, true);
     }
 
     @After
@@ -209,7 +208,7 @@ public void testManyInFlightAsyncCommitsWithCoordinatorDisconnect() throws Excep
         final AtomicInteger responses = new AtomicInteger(0);
 
         for (int i = 0; i < numRequests; i++) {
-            Map offsets = Collections.singletonMap(tp, new OffsetAndMetadata(i));
+            Map offsets = singletonMap(tp, new OffsetAndMetadata(i));
             coordinator.commitOffsetsAsync(offsets, new OffsetCommitCallback() {
                 @Override
                 public void onComplete(Map offsets, Exception exception) {
@@ -237,7 +236,7 @@ public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() throws
         coordinator.ensureCoordinatorReady();
 
         final AtomicBoolean asyncCallbackInvoked = new AtomicBoolean(false);
-        Map offsets = Collections.singletonMap(
+        Map offsets = singletonMap(
                 new TopicPartition("foo", 0), new OffsetCommitRequest.PartitionData(13L, ""));
         consumerClient.send(coordinator.checkAndGetCoordinator(), new OffsetCommitRequest.Builder(groupId, offsets))
                 .compose(new RequestFutureAdapter() {
@@ -379,8 +378,8 @@ public void testNormalJoinGroupLeader() {
         coordinator.ensureCoordinatorReady();
 
         // normal join group
-        Map> memberSubscriptions = Collections.singletonMap(consumerId, singletonList(topic1));
-        partitionAssignor.prepare(Collections.singletonMap(consumerId, singletonList(t1p)));
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
+        partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p)));
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
         client.prepareResponse(new MockClient.RequestMatcher() {
@@ -418,8 +417,8 @@ public void testPatternJoinGroupLeader() {
         coordinator.ensureCoordinatorReady();
 
         // normal join group
-        Map> memberSubscriptions = Collections.singletonMap(consumerId, singletonList(topic1));
-        partitionAssignor.prepare(Collections.singletonMap(consumerId, Arrays.asList(t1p, t2p)));
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
+        partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p, t2p)));
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
         client.prepareResponse(new MockClient.RequestMatcher() {
@@ -521,8 +520,8 @@ public void testWakeupDuringJoin() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        Map> memberSubscriptions = Collections.singletonMap(consumerId, singletonList(topic1));
-        partitionAssignor.prepare(Collections.singletonMap(consumerId, singletonList(t1p)));
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
+        partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p)));
 
         // prepare only the first half of the join and then trigger the wakeup
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
@@ -621,13 +620,7 @@ public void testLeaveGroupOnClose() {
         final String consumerId = "consumer";
 
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
-
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.ensureCoordinatorReady();
-
-        client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
-        coordinator.joinGroupIfNeeded();
+        joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
         final AtomicBoolean received = new AtomicBoolean(false);
         client.prepareResponse(new MockClient.RequestMatcher() {
@@ -648,13 +641,7 @@ public void testMaybeLeaveGroup() {
         final String consumerId = "consumer";
 
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
-
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.ensureCoordinatorReady();
-
-        client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
-        coordinator.joinGroupIfNeeded();
+        joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
         final AtomicBoolean received = new AtomicBoolean(false);
         client.prepareResponse(new MockClient.RequestMatcher() {
@@ -782,8 +769,8 @@ public void testMetadataChangeTriggersRebalance() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        Map> memberSubscriptions = Collections.singletonMap(consumerId, singletonList(topic1));
-        partitionAssignor.prepare(Collections.singletonMap(consumerId, singletonList(t1p)));
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
+        partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p)));
 
         // the leader is responsible for picking up metadata changes and forcing a group rebalance
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
@@ -820,8 +807,8 @@ public void testUpdateMetadataDuringRebalance() {
         coordinator.ensureCoordinatorReady();
 
         // prepare initial rebalance
-        Map> memberSubscriptions = Collections.singletonMap(consumerId, topics);
-        partitionAssignor.prepare(Collections.singletonMap(consumerId, Collections.singletonList(tp1)));
+        Map> memberSubscriptions = singletonMap(consumerId, topics);
+        partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(tp1)));
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
         client.prepareResponse(new MockClient.RequestMatcher() {
@@ -888,7 +875,7 @@ else if (patternSubscribe)
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        Map> memberSubscriptions = Collections.singletonMap(consumerId, singletonList(topic1));
+        Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
         partitionAssignor.prepare(Collections.>emptyMap());
 
         client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
@@ -938,13 +925,8 @@ public void testRejoinGroup() {
 
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
 
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.ensureCoordinatorReady();
-
         // join the group once
-        client.prepareResponse(joinGroupFollowerResponse(1, "consumer", "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
-        coordinator.joinGroupIfNeeded();
+        joinAsFollowerAndReceiveAssignment("consumer", coordinator, singletonList(t1p));
 
         assertEquals(1, rebalanceListener.revokedCount);
         assertTrue(rebalanceListener.revoked.isEmpty());
@@ -1003,10 +985,10 @@ public void testCommitOffsetOnly() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
 
         AtomicBoolean success = new AtomicBoolean(false);
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), callback(success));
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), callback(success));
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertTrue(success.get());
     }
@@ -1030,10 +1012,10 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error)
 
         MockCommitCallback firstCommitCallback = new MockCommitCallback();
         MockCommitCallback secondCommitCallback = new MockCommitCallback();
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), firstCommitCallback);
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), secondCommitCallback);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), firstCommitCallback);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), secondCommitCallback);
 
-        client.respond(offsetCommitResponse(Collections.singletonMap(t1p, error)));
+        respondToOffsetCommitRequest(singletonMap(t1p, 100L), error);
         consumerClient.pollNoWakeup();
         consumerClient.pollNoWakeup(); // second poll since coordinator disconnect is async
         coordinator.invokeCompletedOffsetCommitCallbacks();
@@ -1051,20 +1033,85 @@ public void testAutoCommitDynamicAssignment() {
                 ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, true, true);
 
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
+        joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
+        subscriptions.seek(t1p, 100);
 
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.ensureCoordinatorReady();
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
+        time.sleep(autoCommitIntervalMs);
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertFalse(client.hasPendingResponses());
+    }
 
-        client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
-        coordinator.joinGroupIfNeeded();
+    @Test
+    public void testAutoCommitRetryBackoff() {
+        final String consumerId = "consumer";
+        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
+                ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, true, true);
+        subscriptions.subscribe(singleton(topic1), rebalanceListener);
+        joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
         subscriptions.seek(t1p, 100);
+        time.sleep(autoCommitIntervalMs);
+
+        // Send an offset commit, but let it fail with a retriable error
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NOT_COORDINATOR);
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertTrue(coordinator.coordinatorUnknown());
+
+        // After the disconnect, we should rediscover the coordinator
+        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+
+        subscriptions.seek(t1p, 200);
+
+        // Until the retry backoff has expired, we should not retry the offset commit
+        time.sleep(retryBackoffMs / 2);
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertEquals(0, client.inFlightRequestCount());
+
+        // Once the backoff expires, we should retry
+        time.sleep(retryBackoffMs / 2);
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertEquals(1, client.inFlightRequestCount());
+        respondToOffsetCommitRequest(singletonMap(t1p, 200L), Errors.NONE);
+    }
+
+    @Test
+    public void testAutoCommitAwaitsInterval() {
+        final String consumerId = "consumer";
+        ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors,
+                ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, true, true);
+        subscriptions.subscribe(singleton(topic1), rebalanceListener);
+        joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        subscriptions.seek(t1p, 100);
         time.sleep(autoCommitIntervalMs);
+
+        // Send the offset commit request, but do not respond
         coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
-        assertFalse(client.hasPendingResponses());
+        assertEquals(1, client.inFlightRequestCount());
+
+        time.sleep(autoCommitIntervalMs / 2);
+
+        // Ensure that no additional offset commit is sent
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertEquals(1, client.inFlightRequestCount());
+
+        respondToOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertEquals(0, client.inFlightRequestCount());
+
+        subscriptions.seek(t1p, 200);
+
+        // If we poll again before the auto-commit interval, there should be no new sends
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertEquals(0, client.inFlightRequestCount());
+
+        // After the remainder of the interval passes, we send a new offset commit
+        time.sleep(autoCommitIntervalMs / 2);
+        coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
+        assertEquals(1, client.inFlightRequestCount());
+        respondToOffsetCommitRequest(singletonMap(t1p, 200L), Errors.NONE);
     }
 
     @Test
@@ -1089,7 +1136,7 @@ public void testAutoCommitDynamicAssignmentRebalance() {
 
         subscriptions.seek(t1p, 100);
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
         time.sleep(autoCommitIntervalMs);
         coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
         assertFalse(client.hasPendingResponses());
@@ -1106,7 +1153,7 @@ public void testAutoCommitManualAssignment() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
         time.sleep(autoCommitIntervalMs);
         coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
         assertFalse(client.hasPendingResponses());
@@ -1131,7 +1178,7 @@ public void testAutoCommitManualAssignmentCoordinatorUnknown() {
 
         // sleep only for the retry backoff
         time.sleep(retryBackoffMs);
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
         coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
         assertFalse(client.hasPendingResponses());
     }
@@ -1143,11 +1190,11 @@ public void testCommitOffsetMetadata() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
 
         AtomicBoolean success = new AtomicBoolean(false);
 
-        Map offsets = Collections.singletonMap(t1p, new OffsetAndMetadata(100L, "hello"));
+        Map offsets = singletonMap(t1p, new OffsetAndMetadata(100L, "hello"));
         coordinator.commitOffsetsAsync(offsets, callback(offsets, success));
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertTrue(success.get());
@@ -1158,8 +1205,8 @@ public void testCommitOffsetAsyncWithDefaultCallback() {
         int invokedBeforeTest = mockOffsetCommitCallback.invoked;
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), mockOffsetCommitCallback);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), mockOffsetCommitCallback);
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertEquals(invokedBeforeTest + 1, mockOffsetCommitCallback.invoked);
         assertNull(mockOffsetCommitCallback.exception);
@@ -1170,15 +1217,7 @@ public void testCommitAfterLeaveGroup() {
         // enable auto-assignment
         subscriptions.subscribe(singleton(topic1), rebalanceListener);
 
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.ensureCoordinatorReady();
-
-        client.prepareResponse(joinGroupFollowerResponse(1, "consumer", "leader", Errors.NONE));
-        client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
-
-        client.prepareMetadataUpdate(cluster, Collections.emptySet());
-
-        coordinator.joinGroupIfNeeded();
+        joinAsFollowerAndReceiveAssignment("consumer", coordinator, singletonList(t1p));
 
         // now switch to manual assignment
         client.prepareResponse(new LeaveGroupResponse(Errors.NONE));
@@ -1194,10 +1233,10 @@ public boolean matches(AbstractRequest body) {
                 return commitRequest.memberId().equals(OffsetCommitRequest.DEFAULT_MEMBER_ID) &&
                         commitRequest.generationId() == OffsetCommitRequest.DEFAULT_GENERATION_ID;
             }
-        }, offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        }, offsetCommitResponse(singletonMap(t1p, Errors.NONE)));
 
         AtomicBoolean success = new AtomicBoolean(false);
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), callback(success));
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), callback(success));
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertTrue(success.get());
     }
@@ -1207,8 +1246,8 @@ public void testCommitOffsetAsyncFailedWithDefaultCallback() {
         int invokedBeforeTest = mockOffsetCommitCallback.invoked;
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.COORDINATOR_NOT_AVAILABLE)));
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), mockOffsetCommitCallback);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.COORDINATOR_NOT_AVAILABLE);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), mockOffsetCommitCallback);
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertEquals(invokedBeforeTest + 1, mockOffsetCommitCallback.invoked);
         assertTrue(mockOffsetCommitCallback.exception instanceof RetriableCommitFailedException);
@@ -1221,8 +1260,8 @@ public void testCommitOffsetAsyncCoordinatorNotAvailable() {
 
         // async commit with coordinator not available
         MockCommitCallback cb = new MockCommitCallback();
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.COORDINATOR_NOT_AVAILABLE)));
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), cb);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.COORDINATOR_NOT_AVAILABLE);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), cb);
         coordinator.invokeCompletedOffsetCommitCallbacks();
 
         assertTrue(coordinator.coordinatorUnknown());
@@ -1237,8 +1276,8 @@ public void testCommitOffsetAsyncNotCoordinator() {
 
         // async commit with not coordinator
         MockCommitCallback cb = new MockCommitCallback();
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NOT_COORDINATOR)));
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), cb);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.COORDINATOR_NOT_AVAILABLE);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), cb);
         coordinator.invokeCompletedOffsetCommitCallbacks();
 
         assertTrue(coordinator.coordinatorUnknown());
@@ -1253,8 +1292,8 @@ public void testCommitOffsetAsyncDisconnected() {
 
         // async commit with coordinator disconnected
         MockCommitCallback cb = new MockCommitCallback();
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)), true);
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), cb);
+        prepareOffsetCommitRequestDisconnect(singletonMap(t1p, 100L));
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), cb);
         coordinator.invokeCompletedOffsetCommitCallbacks();
 
         assertTrue(coordinator.coordinatorUnknown());
@@ -1268,10 +1307,10 @@ public void testCommitOffsetSyncNotCoordinator() {
         coordinator.ensureCoordinatorReady();
 
         // sync commit with coordinator disconnected (should connect, get metadata, and then submit the commit request)
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NOT_COORDINATOR)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NOT_COORDINATOR);
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
     }
 
     @Test
@@ -1280,10 +1319,10 @@ public void testCommitOffsetSyncCoordinatorNotAvailable() {
         coordinator.ensureCoordinatorReady();
 
         // sync commit with coordinator disconnected (should connect, get metadata, and then submit the commit request)
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.COORDINATOR_NOT_AVAILABLE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.COORDINATOR_NOT_AVAILABLE);
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
     }
 
     @Test
@@ -1292,10 +1331,10 @@ public void testCommitOffsetSyncCoordinatorDisconnected() {
         coordinator.ensureCoordinatorReady();
 
         // sync commit with coordinator disconnected (should connect, get metadata, and then submit the commit request)
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)), true);
+        prepareOffsetCommitRequestDisconnect(singletonMap(t1p, 100L));
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
     }
 
     @Test
@@ -1307,7 +1346,7 @@ public void testAsyncCommitCallbacksInvokedPriorToSyncCommitCompletion() throws
         final OffsetAndMetadata firstOffset = new OffsetAndMetadata(0L);
         final OffsetAndMetadata secondOffset = new OffsetAndMetadata(1L);
 
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, firstOffset), new OffsetCommitCallback() {
+        coordinator.commitOffsetsAsync(singletonMap(t1p, firstOffset), new OffsetCommitCallback() {
             @Override
             public void onComplete(Map offsets, Exception exception) {
                 committedOffsets.add(firstOffset);
@@ -1318,7 +1357,7 @@ public void onComplete(Map offsets, Exception
         Thread thread = new Thread() {
             @Override
             public void run() {
-                coordinator.commitOffsetsSync(Collections.singletonMap(t1p, secondOffset), 10000);
+                coordinator.commitOffsetsSync(singletonMap(t1p, secondOffset), 10000);
                 committedOffsets.add(secondOffset);
             }
         };
@@ -1326,8 +1365,8 @@ public void run() {
         thread.start();
 
         client.waitForRequests(2, 5000);
-        client.respond(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
-        client.respond(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        respondToOffsetCommitRequest(singletonMap(t1p, firstOffset.offset()), Errors.NONE);
+        respondToOffsetCommitRequest(singletonMap(t1p, secondOffset.offset()), Errors.NONE);
 
         thread.join();
 
@@ -1339,8 +1378,8 @@ public void testCommitUnknownTopicOrPartition() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.UNKNOWN_TOPIC_OR_PARTITION)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.UNKNOWN_TOPIC_OR_PARTITION);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
     }
 
     @Test(expected = OffsetMetadataTooLarge.class)
@@ -1349,8 +1388,8 @@ public void testCommitOffsetMetadataTooLarge() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.OFFSET_METADATA_TOO_LARGE)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.OFFSET_METADATA_TOO_LARGE);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
     }
 
     @Test(expected = CommitFailedException.class)
@@ -1359,8 +1398,8 @@ public void testCommitOffsetIllegalGeneration() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.ILLEGAL_GENERATION)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.ILLEGAL_GENERATION);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
     }
 
     @Test(expected = CommitFailedException.class)
@@ -1369,8 +1408,8 @@ public void testCommitOffsetUnknownMemberId() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.UNKNOWN_MEMBER_ID)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.UNKNOWN_MEMBER_ID);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
     }
 
     @Test(expected = CommitFailedException.class)
@@ -1379,8 +1418,8 @@ public void testCommitOffsetRebalanceInProgress() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
 
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.REBALANCE_IN_PROGRESS)));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.REBALANCE_IN_PROGRESS);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L, "metadata")), Long.MAX_VALUE);
     }
 
     @Test(expected = KafkaException.class)
@@ -1389,21 +1428,21 @@ public void testCommitOffsetSyncCallbackWithNonRetriableException() {
         coordinator.ensureCoordinatorReady();
 
         // sync commit with invalid partitions should throw if we have no callback
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.UNKNOWN_SERVER_ERROR)), false);
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.UNKNOWN_SERVER_ERROR);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), Long.MAX_VALUE);
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void testCommitSyncNegativeOffset() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(-1L)), Long.MAX_VALUE);
+        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(-1L)), Long.MAX_VALUE);
     }
 
     @Test
     public void testCommitAsyncNegativeOffset() {
         int invokedBeforeTest = mockOffsetCommitCallback.invoked;
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(-1L)), mockOffsetCommitCallback);
+        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(-1L)), mockOffsetCommitCallback);
         coordinator.invokeCompletedOffsetCommitCallbacks();
         assertEquals(invokedBeforeTest + 1, mockOffsetCommitCallback.invoked);
         assertTrue(mockOffsetCommitCallback.exception instanceof IllegalArgumentException);
@@ -1413,7 +1452,7 @@ public void testCommitAsyncNegativeOffset() {
     public void testCommitOffsetSyncWithoutFutureGetsCompleted() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady();
-        assertFalse(coordinator.commitOffsetsSync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), 0));
+        assertFalse(coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), 0));
     }
 
     @Test
@@ -1684,7 +1723,7 @@ public void testAutoCommitAfterCoordinatorBackToService() {
         coordinator.markCoordinatorUnknown();
         assertTrue(coordinator.coordinatorUnknown());
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        client.prepareResponse(offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
+        prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE);
 
         // async commit offset should find coordinator
         time.sleep(autoCommitIntervalMs); // sleep for a while to ensure auto commit does happen
@@ -1852,7 +1891,7 @@ private OffsetFetchResponse offsetFetchResponse(Errors topLevelError) {
 
     private OffsetFetchResponse offsetFetchResponse(TopicPartition tp, Errors partitionLevelError, String metadata, long offset) {
         OffsetFetchResponse.PartitionData data = new OffsetFetchResponse.PartitionData(offset, metadata, partitionLevelError);
-        return new OffsetFetchResponse(Errors.NONE, Collections.singletonMap(tp, data));
+        return new OffsetFetchResponse(Errors.NONE, singletonMap(tp, data));
     }
 
     private OffsetCommitCallback callback(final AtomicBoolean success) {
@@ -1865,6 +1904,67 @@ public void onComplete(Map offsets, Exception
         };
     }
 
+    private void joinAsFollowerAndReceiveAssignment(String consumerId,
+                                                    ConsumerCoordinator coordinator,
+                                                    List assignment) {
+        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.ensureCoordinatorReady();
+        client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
+        client.prepareResponse(syncGroupResponse(assignment, Errors.NONE));
+        coordinator.joinGroupIfNeeded();
+    }
+
+    private void prepareOffsetCommitRequest(Map expectedOffsets, Errors error) {
+        prepareOffsetCommitRequest(expectedOffsets, error, false);
+    }
+
+    private void prepareOffsetCommitRequestDisconnect(Map expectedOffsets) {
+        prepareOffsetCommitRequest(expectedOffsets, Errors.NONE, true);
+    }
+
+    private void prepareOffsetCommitRequest(final Map expectedOffsets,
+                                            Errors error,
+                                            boolean disconnected) {
+        Map errors = partitionErrors(expectedOffsets.keySet(), error);
+        client.prepareResponse(offsetCommitRequestMatcher(expectedOffsets), offsetCommitResponse(errors), disconnected);
+    }
+
+    private Map partitionErrors(Collection partitions, Errors error) {
+        final Map errors = new HashMap<>();
+        for (TopicPartition partition : partitions) {
+            errors.put(partition, error);
+        }
+        return errors;
+    }
+
+    private void respondToOffsetCommitRequest(final Map expectedOffsets, Errors error) {
+        Map errors = partitionErrors(expectedOffsets.keySet(), error);
+        client.respond(offsetCommitRequestMatcher(expectedOffsets), offsetCommitResponse(errors));
+    }
+
+    private MockClient.RequestMatcher offsetCommitRequestMatcher(final Map expectedOffsets) {
+        return new MockClient.RequestMatcher() {
+            @Override
+            public boolean matches(AbstractRequest body) {
+                OffsetCommitRequest req = (OffsetCommitRequest) body;
+                Map offsets = req.offsetData();
+                if (offsets.size() != expectedOffsets.size())
+                    return false;
+
+                for (Map.Entry expectedOffset : expectedOffsets.entrySet()) {
+                    if (!offsets.containsKey(expectedOffset.getKey()))
+                        return false;
+
+                    OffsetCommitRequest.PartitionData offsetCommitData = offsets.get(expectedOffset.getKey());
+                    if (offsetCommitData.offset != expectedOffset.getValue())
+                        return false;
+                }
+
+                return true;
+            }
+        };
+    }
+
     private OffsetCommitCallback callback(final Map expectedOffsets,
                                           final AtomicBoolean success) {
         return new OffsetCommitCallback() {
@@ -1876,7 +1976,6 @@ public void onComplete(Map offsets, Exception
         };
     }
 
-
     private static class MockCommitCallback implements OffsetCommitCallback {
         public int invoked = 0;
         public Exception exception = null;

From 2a4ba75e1338eaa97da87077330b43d7448a18bc Mon Sep 17 00:00:00 2001
From: "Matthias J. Sax" 
Date: Mon, 5 Mar 2018 10:56:42 -0800
Subject: [PATCH 0112/1847] KAFKA-6054: Code cleanup to prepare the actual fix
 for an upgrade path (#4630)

Author: Matthias J. Sax 

Reviewers: Bill Bejeck , John Roesler , Guozhang Wang 
---
 .../apache/kafka/streams/StreamsConfig.java   |   4 +-
 .../internals/StreamsPartitionAssignor.java   | 230 ++++++++++-------
 .../internals/assignment/AssignmentInfo.java  | 236 +++++++++++-------
 .../assignment/SubscriptionInfo.java          | 224 ++++++++++++-----
 .../QueryableStateIntegrationTest.java        |   1 -
 .../StreamsPartitionAssignorTest.java         |  95 ++++---
 .../assignment/AssignmentInfoTest.java        |  19 +-
 .../assignment/SubscriptionInfoTest.java      |  10 +-
 8 files changed, 530 insertions(+), 289 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index 6b3626101bdb6..47becfc239b21 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -226,11 +226,11 @@ public class StreamsConfig extends AbstractConfig {
     public static final String DEFAULT_KEY_SERDE_CLASS_CONFIG = "default.key.serde";
     private static final String DEFAULT_KEY_SERDE_CLASS_DOC = " Default serializer / deserializer class for key that implements the org.apache.kafka.common.serialization.Serde interface.";
 
-    /** {@code default timestamp.extractor} */
+    /** {@code default.timestamp.extractor} */
     public static final String DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG = "default.timestamp.extractor";
     private static final String DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_DOC = "Default timestamp extractor class that implements the org.apache.kafka.streams.processor.TimestampExtractor interface.";
 
-    /** {@code default value.serde} */
+    /** {@code default.value.serde} */
     public static final String DEFAULT_VALUE_SERDE_CLASS_CONFIG = "default.value.serde";
     private static final String DEFAULT_VALUE_SERDE_CLASS_DOC = "Default serializer / deserializer class for value that implements the org.apache.kafka.common.serialization.Serde interface.";
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
index 9aa0e94c8c1ba..71a84b2ca73de 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java
@@ -66,7 +66,8 @@ private static class AssignedPartition implements Comparable
         public final TaskId taskId;
         public final TopicPartition partition;
 
-        AssignedPartition(final TaskId taskId, final TopicPartition partition) {
+        AssignedPartition(final TaskId taskId,
+                          final TopicPartition partition) {
             this.taskId = taskId;
             this.partition = partition;
         }
@@ -77,11 +78,11 @@ public int compareTo(final AssignedPartition that) {
         }
 
         @Override
-        public boolean equals(Object o) {
+        public boolean equals(final Object o) {
             if (!(o instanceof AssignedPartition)) {
                 return false;
             }
-            AssignedPartition other = (AssignedPartition) o;
+            final AssignedPartition other = (AssignedPartition) o;
             return compareTo(other) == 0;
         }
 
@@ -104,8 +105,9 @@ private static class ClientMetadata {
                 final String host = getHost(endPoint);
                 final Integer port = getPort(endPoint);
 
-                if (host == null || port == null)
+                if (host == null || port == null) {
                     throw new ConfigException(String.format("Error parsing host address %s. Expected format host:port.", endPoint));
+                }
 
                 hostInfo = new HostInfo(host, port);
             } else {
@@ -119,10 +121,11 @@ private static class ClientMetadata {
             state = new ClientState();
         }
 
-        void addConsumer(final String consumerMemberId, final SubscriptionInfo info) {
+        void addConsumer(final String consumerMemberId,
+                         final SubscriptionInfo info) {
             consumers.add(consumerMemberId);
-            state.addPreviousActiveTasks(info.prevTasks);
-            state.addPreviousStandbyTasks(info.standbyTasks);
+            state.addPreviousActiveTasks(info.prevTasks());
+            state.addPreviousStandbyTasks(info.standbyTasks());
             state.incrementCapacity();
         }
 
@@ -157,8 +160,9 @@ public String toString() {
 
     private static final Comparator PARTITION_COMPARATOR = new Comparator() {
         @Override
-        public int compare(TopicPartition p1, TopicPartition p2) {
-            int result = p1.topic().compareTo(p2.topic());
+        public int compare(final TopicPartition p1,
+                           final TopicPartition p2) {
+            final int result = p1.topic().compareTo(p2.topic());
 
             if (result != 0) {
                 return result;
@@ -194,15 +198,15 @@ public void configure(final Map configs) {
 
         final Object o = configs.get(StreamsConfig.InternalConfig.TASK_MANAGER_FOR_PARTITION_ASSIGNOR);
         if (o == null) {
-            KafkaException ex = new KafkaException("TaskManager is not specified");
-            log.error(ex.getMessage(), ex);
-            throw ex;
+            final KafkaException fatalException = new KafkaException("TaskManager is not specified");
+            log.error(fatalException.getMessage(), fatalException);
+            throw fatalException;
         }
 
         if (!(o instanceof TaskManager)) {
-            KafkaException ex = new KafkaException(String.format("%s is not an instance of %s", o.getClass().getName(), TaskManager.class.getName()));
-            log.error(ex.getMessage(), ex);
-            throw ex;
+            final KafkaException fatalException = new KafkaException(String.format("%s is not an instance of %s", o.getClass().getName(), TaskManager.class.getName()));
+            log.error(fatalException.getMessage(), fatalException);
+            throw fatalException;
         }
 
         taskManager = (TaskManager) o;
@@ -214,14 +218,14 @@ public void configure(final Map configs) {
         final String userEndPoint = streamsConfig.getString(StreamsConfig.APPLICATION_SERVER_CONFIG);
         if (userEndPoint != null && !userEndPoint.isEmpty()) {
             try {
-                String host = getHost(userEndPoint);
-                Integer port = getPort(userEndPoint);
+                final String host = getHost(userEndPoint);
+                final Integer port = getPort(userEndPoint);
 
                 if (host == null || port == null)
                     throw new ConfigException(String.format("%s Config %s isn't in the correct format. Expected a host:port pair" +
                                     " but received %s",
                             logPrefix, StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint));
-            } catch (NumberFormatException nfe) {
+            } catch (final NumberFormatException nfe) {
                 throw new ConfigException(String.format("%s Invalid port supplied in %s for config %s",
                         logPrefix, userEndPoint, StreamsConfig.APPLICATION_SERVER_CONFIG));
             }
@@ -240,7 +244,7 @@ public String name() {
     }
 
     @Override
-    public Subscription subscription(Set topics) {
+    public Subscription subscription(final Set topics) {
         // Adds the following information to subscription
         // 1. Client UUID (a unique id assigned to an instance of KafkaStreams)
         // 2. Task ids of previously running tasks
@@ -249,7 +253,11 @@ public Subscription subscription(Set topics) {
         final Set previousActiveTasks = taskManager.prevActiveTaskIds();
         final Set standbyTasks = taskManager.cachedTasksIds();
         standbyTasks.removeAll(previousActiveTasks);
-        final SubscriptionInfo data = new SubscriptionInfo(taskManager.processId(), previousActiveTasks, standbyTasks, this.userEndPoint);
+        final SubscriptionInfo data = new SubscriptionInfo(
+            taskManager.processId(),
+            previousActiveTasks,
+            standbyTasks,
+            this.userEndPoint);
 
         taskManager.updateSubscriptionsFromMetadata(topics);
 
@@ -277,22 +285,32 @@ public Subscription subscription(Set topics) {
      * 3. within each client, tasks are assigned to consumer clients in round-robin manner.
      */
     @Override
-    public Map assign(Cluster metadata, Map subscriptions) {
+    public Map assign(final Cluster metadata,
+                                          final Map subscriptions) {
         // construct the client metadata from the decoded subscription info
-        Map clientsMetadata = new HashMap<>();
-
-        for (Map.Entry entry : subscriptions.entrySet()) {
-            String consumerId = entry.getKey();
-            Subscription subscription = entry.getValue();
-
-            SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
+        final Map clientsMetadata = new HashMap<>();
+
+        int minUserMetadataVersion = SubscriptionInfo.LATEST_SUPPORTED_VERSION;
+        for (final Map.Entry entry : subscriptions.entrySet()) {
+            final String consumerId = entry.getKey();
+            final Subscription subscription = entry.getValue();
+
+            final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
+            final int usedVersion = info.version();
+            if (usedVersion > SubscriptionInfo.LATEST_SUPPORTED_VERSION) {
+                throw new IllegalStateException("Unknown metadata version: " + usedVersion
+                    + "; latest supported version: " + SubscriptionInfo.LATEST_SUPPORTED_VERSION);
+            }
+            if (usedVersion < minUserMetadataVersion) {
+                minUserMetadataVersion = usedVersion;
+            }
 
             // create the new client metadata if necessary
-            ClientMetadata clientMetadata = clientsMetadata.get(info.processId);
+            ClientMetadata clientMetadata = clientsMetadata.get(info.processId());
 
             if (clientMetadata == null) {
-                clientMetadata = new ClientMetadata(info.userEndPoint);
-                clientsMetadata.put(info.processId, clientMetadata);
+                clientMetadata = new ClientMetadata(info.userEndPoint());
+                clientsMetadata.put(info.processId(), clientMetadata);
             }
 
             // add the consumer to the client
@@ -309,8 +327,8 @@ public Map assign(Cluster metadata, Map topicGroups = taskManager.builder().topicGroups();
 
         final Map repartitionTopicMetadata = new HashMap<>();
-        for (InternalTopologyBuilder.TopicsInfo topicsInfo : topicGroups.values()) {
-            for (InternalTopicConfig topic: topicsInfo.repartitionSourceTopics.values()) {
+        for (final InternalTopologyBuilder.TopicsInfo topicsInfo : topicGroups.values()) {
+            for (final InternalTopicConfig topic: topicsInfo.repartitionSourceTopics.values()) {
                 repartitionTopicMetadata.put(topic.name(), new InternalTopicMetadata(topic));
             }
         }
@@ -319,13 +337,13 @@ public Map assign(Cluster metadata, Map otherSinkTopics = otherTopicsInfo.sinkTopics;
 
                             if (otherSinkTopics.contains(topicName)) {
@@ -375,7 +393,7 @@ public Map assign(Cluster metadata, Map allRepartitionTopicPartitions = new HashMap<>();
-        for (Map.Entry entry : repartitionTopicMetadata.entrySet()) {
+        for (final Map.Entry entry : repartitionTopicMetadata.entrySet()) {
             final String topic = entry.getKey();
             final int numPartitions = entry.getValue().numPartitions;
 
@@ -395,7 +413,7 @@ public Map assign(Cluster metadata, Map allSourceTopics = new HashSet<>();
         final Map> sourceTopicsByGroup = new HashMap<>();
-        for (Map.Entry entry : topicGroups.entrySet()) {
+        for (final Map.Entry entry : topicGroups.entrySet()) {
             allSourceTopics.addAll(entry.getValue().sourceTopics);
             sourceTopicsByGroup.put(entry.getKey(), entry.getValue().sourceTopics);
         }
@@ -405,9 +423,9 @@ public Map assign(Cluster metadata, Map allAssignedPartitions = new HashSet<>();
         final Map> tasksByTopicGroup = new HashMap<>();
-        for (Map.Entry> entry : partitionsForTask.entrySet()) {
+        for (final Map.Entry> entry : partitionsForTask.entrySet()) {
             final Set partitions = entry.getValue();
-            for (TopicPartition partition : partitions) {
+            for (final TopicPartition partition : partitions) {
                 if (allAssignedPartitions.contains(partition)) {
                     log.warn("Partition {} is assigned to more than one tasks: {}", partition, partitionsForTask);
                 }
@@ -422,10 +440,10 @@ public Map assign(Cluster metadata, Map partitionInfoList = fullMetadata.partitionsForTopic(topic);
             if (!partitionInfoList.isEmpty()) {
-                for (PartitionInfo partitionInfo : partitionInfoList) {
+                for (final PartitionInfo partitionInfo : partitionInfoList) {
                     final TopicPartition partition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
                     if (!allAssignedPartitions.contains(partition)) {
                         log.warn("Partition {} is not assigned to any tasks: {}", partition, partitionsForTask);
@@ -438,15 +456,15 @@ public Map assign(Cluster metadata, Map changelogTopicMetadata = new HashMap<>();
-        for (Map.Entry entry : topicGroups.entrySet()) {
+        for (final Map.Entry entry : topicGroups.entrySet()) {
             final int topicGroupId = entry.getKey();
             final Map stateChangelogTopics = entry.getValue().stateChangelogTopics;
 
-            for (InternalTopicConfig topicConfig : stateChangelogTopics.values()) {
+            for (final InternalTopicConfig topicConfig : stateChangelogTopics.values()) {
                 // the expected number of partitions is the max value of TaskId.partition + 1
                 int numPartitions = UNKNOWN;
                 if (tasksByTopicGroup.get(topicGroupId) != null) {
-                    for (TaskId task : tasksByTopicGroup.get(topicGroupId)) {
+                    for (final TaskId task : tasksByTopicGroup.get(topicGroupId)) {
                         if (numPartitions < task.partition + 1)
                             numPartitions = task.partition + 1;
                     }
@@ -468,7 +486,7 @@ public Map assign(Cluster metadata, Map states = new HashMap<>();
-        for (Map.Entry entry : clientsMetadata.entrySet()) {
+        for (final Map.Entry entry : clientsMetadata.entrySet()) {
             states.put(entry.getKey(), entry.getValue().state);
         }
 
@@ -484,25 +502,27 @@ public Map assign(Cluster metadata, Map> partitionsByHostState = new HashMap<>();
-        for (Map.Entry entry : clientsMetadata.entrySet()) {
-            final HostInfo hostInfo = entry.getValue().hostInfo;
+        if (minUserMetadataVersion == 2) {
+            for (final Map.Entry entry : clientsMetadata.entrySet()) {
+                final HostInfo hostInfo = entry.getValue().hostInfo;
 
-            if (hostInfo != null) {
-                final Set topicPartitions = new HashSet<>();
-                final ClientState state = entry.getValue().state;
+                if (hostInfo != null) {
+                    final Set topicPartitions = new HashSet<>();
+                    final ClientState state = entry.getValue().state;
 
-                for (final TaskId id : state.activeTasks()) {
-                    topicPartitions.addAll(partitionsForTask.get(id));
-                }
+                    for (final TaskId id : state.activeTasks()) {
+                        topicPartitions.addAll(partitionsForTask.get(id));
+                    }
 
-                partitionsByHostState.put(hostInfo, topicPartitions);
+                    partitionsByHostState.put(hostInfo, topicPartitions);
+                }
             }
         }
         taskManager.setPartitionsByHostState(partitionsByHostState);
 
         // within the client, distribute tasks to its owned consumers
         final Map assignment = new HashMap<>();
-        for (Map.Entry entry : clientsMetadata.entrySet()) {
+        for (final Map.Entry entry : clientsMetadata.entrySet()) {
             final Set consumers = entry.getValue().consumers;
             final ClientState state = entry.getValue().state;
 
@@ -511,7 +531,7 @@ public Map assign(Cluster metadata, Map> standby = new HashMap<>();
                 final ArrayList assignedPartitions = new ArrayList<>();
 
@@ -540,13 +560,15 @@ public Map assign(Cluster metadata, Map active = new ArrayList<>();
                 final List activePartitions = new ArrayList<>();
-                for (AssignedPartition partition : assignedPartitions) {
+                for (final AssignedPartition partition : assignedPartitions) {
                     active.add(partition.taskId);
                     activePartitions.add(partition.partition);
                 }
 
                 // finally, encode the assignment before sending back to coordinator
-                assignment.put(consumer, new Assignment(activePartitions, new AssignmentInfo(active, standby, partitionsByHostState).encode()));
+                assignment.put(consumer, new Assignment(
+                    activePartitions,
+                    new AssignmentInfo(minUserMetadataVersion, active, standby, partitionsByHostState).encode()));
             }
         }
 
@@ -577,26 +599,54 @@ List> interleaveTasksByGroupId(final Collection taskIds, fi
      * @throws TaskAssignmentException if there is no task id for one of the partitions specified
      */
     @Override
-    public void onAssignment(Assignment assignment) {
-        List partitions = new ArrayList<>(assignment.partitions());
+    public void onAssignment(final Assignment assignment) {
+        final List partitions = new ArrayList<>(assignment.partitions());
         Collections.sort(partitions, PARTITION_COMPARATOR);
 
-        AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
+        final AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
+        final int usedVersion = info.version();
 
-        Map> activeTasks = new HashMap<>();
+        // version 1 field
+        final Map> activeTasks = new HashMap<>();
+        // version 2 fields
+        final Map topicToPartitionInfo = new HashMap<>();
+        final Map> partitionsByHost;
+
+        switch (usedVersion) {
+            case 1:
+                processVersionOneAssignment(info, partitions, activeTasks);
+                partitionsByHost = Collections.emptyMap();
+                break;
+            case 2:
+                processVersionTwoAssignment(info, partitions, activeTasks, topicToPartitionInfo);
+                partitionsByHost = info.partitionsByHost();
+                break;
+            default:
+                throw new IllegalStateException("Unknown metadata version: " + usedVersion
+                    + "; latest supported version: " + AssignmentInfo.LATEST_SUPPORTED_VERSION);
+        }
 
+        taskManager.setClusterMetadata(Cluster.empty().withPartitions(topicToPartitionInfo));
+        taskManager.setPartitionsByHostState(partitionsByHost);
+        taskManager.setAssignmentMetadata(activeTasks, info.standbyTasks());
+        taskManager.updateSubscriptionsFromAssignment(partitions);
+    }
+
+    private void processVersionOneAssignment(final AssignmentInfo info,
+                                             final List partitions,
+                                             final Map> activeTasks) {
         // the number of assigned partitions should be the same as number of active tasks, which
         // could be duplicated if one task has more than one assigned partitions
-        if (partitions.size() != info.activeTasks.size()) {
+        if (partitions.size() != info.activeTasks().size()) {
             throw new TaskAssignmentException(
-                    String.format("%sNumber of assigned partitions %d is not equal to the number of active taskIds %d" +
-                            ", assignmentInfo=%s", logPrefix, partitions.size(), info.activeTasks.size(), info.toString())
+                String.format("%sNumber of assigned partitions %d is not equal to the number of active taskIds %d" +
+                    ", assignmentInfo=%s", logPrefix, partitions.size(), info.activeTasks().size(), info.toString())
             );
         }
 
         for (int i = 0; i < partitions.size(); i++) {
-            TopicPartition partition = partitions.get(i);
-            TaskId id = info.activeTasks.get(i);
+            final TopicPartition partition = partitions.get(i);
+            final TaskId id = info.activeTasks().get(i);
 
             Set assignedPartitions = activeTasks.get(id);
             if (assignedPartitions == null) {
@@ -605,23 +655,23 @@ public void onAssignment(Assignment assignment) {
             }
             assignedPartitions.add(partition);
         }
+    }
 
-        final Map topicToPartitionInfo = new HashMap<>();
-        for (Set value : info.partitionsByHost.values()) {
-            for (TopicPartition topicPartition : value) {
-                topicToPartitionInfo.put(topicPartition, new PartitionInfo(topicPartition.topic(),
-                        topicPartition.partition(),
-                        null,
-                        new Node[0],
-                        new Node[0]));
+    private void processVersionTwoAssignment(final AssignmentInfo info,
+                                             final List partitions,
+                                             final Map> activeTasks,
+                                             final Map topicToPartitionInfo) {
+        processVersionOneAssignment(info, partitions, activeTasks);
+
+        // process partitions by host
+        final Map> partitionsByHost = info.partitionsByHost();
+        for (final Set value : partitionsByHost.values()) {
+            for (final TopicPartition topicPartition : value) {
+                topicToPartitionInfo.put(
+                    topicPartition,
+                    new PartitionInfo(topicPartition.topic(), topicPartition.partition(), null, new Node[0], new Node[0]));
             }
         }
-
-        taskManager.setClusterMetadata(Cluster.empty().withPartitions(topicToPartitionInfo));
-        taskManager.setPartitionsByHostState(info.partitionsByHost);
-        taskManager.setAssignmentMetadata(activeTasks, info.standbyTasks);
-
-        taskManager.updateSubscriptionsFromAssignment(partitions);
     }
 
     /**
@@ -658,10 +708,10 @@ private void prepareTopic(final Map topicPartitio
         log.debug("Completed validating internal topics in partition assignor.");
     }
 
-    private void ensureCopartitioning(Collection> copartitionGroups,
-                                      Map allRepartitionTopicsNumPartitions,
-                                      Cluster metadata) {
-        for (Set copartitionGroup : copartitionGroups) {
+    private void ensureCopartitioning(final Collection> copartitionGroups,
+                                      final Map allRepartitionTopicsNumPartitions,
+                                      final Cluster metadata) {
+        for (final Set copartitionGroup : copartitionGroups) {
             copartitionedTopicsValidator.validate(copartitionGroup, allRepartitionTopicsNumPartitions, metadata);
         }
     }
@@ -677,7 +727,7 @@ public static class SubscriptionUpdates {
 
         private final Set updatedTopicSubscriptions = new HashSet<>();
 
-        public void updateTopics(Collection topicNames) {
+        public void updateTopics(final Collection topicNames) {
             updatedTopicSubscriptions.clear();
             updatedTopicSubscriptions.addAll(topicNames);
         }
@@ -735,7 +785,7 @@ void validate(final Set copartitionGroup,
             // if all topics for this co-partition group is repartition topics,
             // then set the number of partitions to be the maximum of the number of partitions.
             if (numPartitions == UNKNOWN) {
-                for (Map.Entry entry: allRepartitionTopicsNumPartitions.entrySet()) {
+                for (final Map.Entry entry: allRepartitionTopicsNumPartitions.entrySet()) {
                     if (copartitionGroup.contains(entry.getKey())) {
                         final int partitions = entry.getValue().numPartitions;
                         if (partitions > numPartitions) {
@@ -745,7 +795,7 @@ void validate(final Set copartitionGroup,
                 }
             }
             // enforce co-partitioning restrictions to repartition topics by updating their number of partitions
-            for (Map.Entry entry : allRepartitionTopicsNumPartitions.entrySet()) {
+            for (final Map.Entry entry : allRepartitionTopicsNumPartitions.entrySet()) {
                 if (copartitionGroup.contains(entry.getKey())) {
                     entry.getValue().numPartitions = numPartitions;
                 }
@@ -755,7 +805,7 @@ void validate(final Set copartitionGroup,
     }
 
     // following functions are for test only
-    void setInternalTopicManager(InternalTopicManager internalTopicManager) {
+    void setInternalTopicManager(final InternalTopicManager internalTopicManager) {
         this.internalTopicManager = internalTopicManager;
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
index 8607472c281f2..c8df7498755bb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfo.java
@@ -39,76 +39,123 @@
 public class AssignmentInfo {
 
     private static final Logger log = LoggerFactory.getLogger(AssignmentInfo.class);
-    /**
-     * A new field was added, partitionsByHost. CURRENT_VERSION
-     * is required so we can decode the previous version. For example, this may occur
-     * during a rolling upgrade
-     */
-    private static final int CURRENT_VERSION = 2;
-    public final int version;
-    public final List activeTasks; // each element corresponds to a partition
-    public final Map> standbyTasks;
-    public final Map> partitionsByHost;
 
-    public AssignmentInfo(List activeTasks, Map> standbyTasks,
-                          Map> hostState) {
-        this(CURRENT_VERSION, activeTasks, standbyTasks, hostState);
+    public static final int LATEST_SUPPORTED_VERSION = 2;
+
+    private final int usedVersion;
+    private List activeTasks;
+    private Map> standbyTasks;
+    private Map> partitionsByHost;
+
+    private AssignmentInfo(final int version) {
+        this.usedVersion = version;
+    }
+
+    public AssignmentInfo(final List activeTasks,
+                          final Map> standbyTasks,
+                          final Map> hostState) {
+        this(LATEST_SUPPORTED_VERSION, activeTasks, standbyTasks, hostState);
     }
 
-    protected AssignmentInfo(int version, List activeTasks, Map> standbyTasks,
-                             Map> hostState) {
-        this.version = version;
+    public AssignmentInfo(final int version,
+                          final List activeTasks,
+                          final Map> standbyTasks,
+                          final Map> hostState) {
+        this.usedVersion = version;
         this.activeTasks = activeTasks;
         this.standbyTasks = standbyTasks;
         this.partitionsByHost = hostState;
     }
 
+    public int version() {
+        return usedVersion;
+    }
+
+    public List activeTasks() {
+        return activeTasks;
+    }
+
+    public Map> standbyTasks() {
+        return standbyTasks;
+    }
+
+    public Map> partitionsByHost() {
+        return partitionsByHost;
+    }
+
     /**
      * @throws TaskAssignmentException if method fails to encode the data, e.g., if there is an
      * IO exception during encoding
      */
     public ByteBuffer encode() {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        DataOutputStream out = new DataOutputStream(baos);
-
-        try {
-            // Encode version
-            out.writeInt(version);
-            // Encode active tasks
-            out.writeInt(activeTasks.size());
-            for (TaskId id : activeTasks) {
-                id.writeTo(out);
-            }
-            // Encode standby tasks
-            out.writeInt(standbyTasks.size());
-            for (Map.Entry> entry : standbyTasks.entrySet()) {
-                TaskId id = entry.getKey();
-                id.writeTo(out);
-
-                Set partitions = entry.getValue();
-                writeTopicPartitions(out, partitions);
-            }
-            out.writeInt(partitionsByHost.size());
-            for (Map.Entry> entry : partitionsByHost
-                    .entrySet()) {
-                final HostInfo hostInfo = entry.getKey();
-                out.writeUTF(hostInfo.host());
-                out.writeInt(hostInfo.port());
-                writeTopicPartitions(out, entry.getValue());
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+        try (final DataOutputStream out = new DataOutputStream(baos)) {
+            switch (usedVersion) {
+                case 1:
+                    encodeVersionOne(out);
+                    break;
+                case 2:
+                    encodeVersionTwo(out);
+                    break;
+                default:
+                    throw new IllegalStateException("Unknown metadata version: " + usedVersion
+                        + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
             }
 
             out.flush();
             out.close();
 
             return ByteBuffer.wrap(baos.toByteArray());
-        } catch (IOException ex) {
+        } catch (final IOException ex) {
             throw new TaskAssignmentException("Failed to encode AssignmentInfo", ex);
         }
     }
 
-    private void writeTopicPartitions(DataOutputStream out, Set partitions) throws IOException {
+    private void encodeVersionOne(final DataOutputStream out) throws IOException {
+        out.writeInt(1); // version
+        encodeActiveAndStandbyTaskAssignment(out);
+    }
+
+    private void encodeActiveAndStandbyTaskAssignment(final DataOutputStream out) throws IOException {
+        // encode active tasks
+        out.writeInt(activeTasks.size());
+        for (final TaskId id : activeTasks) {
+            id.writeTo(out);
+        }
+
+        // encode standby tasks
+        out.writeInt(standbyTasks.size());
+        for (final Map.Entry> entry : standbyTasks.entrySet()) {
+            final TaskId id = entry.getKey();
+            id.writeTo(out);
+
+            final Set partitions = entry.getValue();
+            writeTopicPartitions(out, partitions);
+        }
+    }
+
+    private void encodeVersionTwo(final DataOutputStream out) throws IOException {
+        out.writeInt(2); // version
+        encodeActiveAndStandbyTaskAssignment(out);
+        encodePartitionsByHost(out);
+    }
+
+    private void encodePartitionsByHost(final DataOutputStream out) throws IOException {
+        // encode partitions by host
+        out.writeInt(partitionsByHost.size());
+        for (final Map.Entry> entry : partitionsByHost.entrySet()) {
+            final HostInfo hostInfo = entry.getKey();
+            out.writeUTF(hostInfo.host());
+            out.writeInt(hostInfo.port());
+            writeTopicPartitions(out, entry.getValue());
+        }
+    }
+
+    private void writeTopicPartitions(final DataOutputStream out,
+                                      final Set partitions) throws IOException {
         out.writeInt(partitions.size());
-        for (TopicPartition partition : partitions) {
+        for (final TopicPartition partition : partitions) {
             out.writeUTF(partition.topic());
             out.writeInt(partition.partition());
         }
@@ -117,52 +164,69 @@ private void writeTopicPartitions(DataOutputStream out, Set part
     /**
      * @throws TaskAssignmentException if method fails to decode the data or if the data version is unknown
      */
-    public static AssignmentInfo decode(ByteBuffer data) {
+    public static AssignmentInfo decode(final ByteBuffer data) {
         // ensure we are at the beginning of the ByteBuffer
         data.rewind();
 
-        try (DataInputStream in = new DataInputStream(new ByteBufferInputStream(data))) {
-            // Decode version
-            int version = in.readInt();
-            if (version < 0 || version > CURRENT_VERSION) {
-                TaskAssignmentException ex = new TaskAssignmentException("Unknown assignment data version: " + version);
-                log.error(ex.getMessage(), ex);
-                throw ex;
-            }
+        try (final DataInputStream in = new DataInputStream(new ByteBufferInputStream(data))) {
+            // decode used version
+            final int usedVersion = in.readInt();
+            final AssignmentInfo assignmentInfo = new AssignmentInfo(usedVersion);
 
-            // Decode active tasks
-            int count = in.readInt();
-            List activeTasks = new ArrayList<>(count);
-            for (int i = 0; i < count; i++) {
-                activeTasks.add(TaskId.readFrom(in));
-            }
-            // Decode standby tasks
-            count = in.readInt();
-            Map> standbyTasks = new HashMap<>(count);
-            for (int i = 0; i < count; i++) {
-                TaskId id = TaskId.readFrom(in);
-                standbyTasks.put(id, readTopicPartitions(in));
+            switch (usedVersion) {
+                case 1:
+                    decodeVersionOneData(assignmentInfo, in);
+                    break;
+                case 2:
+                    decodeVersionTwoData(assignmentInfo, in);
+                    break;
+                default:
+                    TaskAssignmentException fatalException = new TaskAssignmentException("Unable to decode subscription data: " +
+                        "used version: " + usedVersion + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
+                    log.error(fatalException.getMessage(), fatalException);
+                    throw fatalException;
             }
 
-            Map> hostStateToTopicPartitions = new HashMap<>();
-            if (version == CURRENT_VERSION) {
-                int numEntries = in.readInt();
-                for (int i = 0; i < numEntries; i++) {
-                    HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt());
-                    hostStateToTopicPartitions.put(hostInfo, readTopicPartitions(in));
-                }
-            }
+            return assignmentInfo;
+        } catch (final IOException ex) {
+            throw new TaskAssignmentException("Failed to decode AssignmentInfo", ex);
+        }
+    }
+
+    private static void decodeVersionOneData(final AssignmentInfo assignmentInfo,
+                                             final DataInputStream in) throws IOException {
+        // decode active tasks
+        int count = in.readInt();
+        assignmentInfo.activeTasks = new ArrayList<>(count);
+        for (int i = 0; i < count; i++) {
+            assignmentInfo.activeTasks.add(TaskId.readFrom(in));
+        }
 
-            return new AssignmentInfo(activeTasks, standbyTasks, hostStateToTopicPartitions);
+        // decode standby tasks
+        count = in.readInt();
+        assignmentInfo.standbyTasks = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            TaskId id = TaskId.readFrom(in);
+            assignmentInfo.standbyTasks.put(id, readTopicPartitions(in));
+        }
+    }
 
-        } catch (IOException ex) {
-            throw new TaskAssignmentException("Failed to decode AssignmentInfo", ex);
+    private static void decodeVersionTwoData(final AssignmentInfo assignmentInfo,
+                                             final DataInputStream in) throws IOException {
+        decodeVersionOneData(assignmentInfo, in);
+
+        // decode partitions by host
+        assignmentInfo.partitionsByHost = new HashMap<>();
+        final int numEntries = in.readInt();
+        for (int i = 0; i < numEntries; i++) {
+            final HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt());
+            assignmentInfo.partitionsByHost.put(hostInfo, readTopicPartitions(in));
         }
     }
 
-    private static Set readTopicPartitions(DataInputStream in) throws IOException {
-        int numPartitions = in.readInt();
-        Set partitions = new HashSet<>(numPartitions);
+    private static Set readTopicPartitions(final DataInputStream in) throws IOException {
+        final int numPartitions = in.readInt();
+        final Set partitions = new HashSet<>(numPartitions);
         for (int j = 0; j < numPartitions; j++) {
             partitions.add(new TopicPartition(in.readUTF(), in.readInt()));
         }
@@ -171,14 +235,14 @@ private static Set readTopicPartitions(DataInputStream in) throw
 
     @Override
     public int hashCode() {
-        return version ^ activeTasks.hashCode() ^ standbyTasks.hashCode() ^ partitionsByHost.hashCode();
+        return usedVersion ^ activeTasks.hashCode() ^ standbyTasks.hashCode() ^ partitionsByHost.hashCode();
     }
 
     @Override
-    public boolean equals(Object o) {
+    public boolean equals(final Object o) {
         if (o instanceof AssignmentInfo) {
-            AssignmentInfo other = (AssignmentInfo) o;
-            return this.version == other.version &&
+            final AssignmentInfo other = (AssignmentInfo) o;
+            return this.usedVersion == other.usedVersion &&
                     this.activeTasks.equals(other.activeTasks) &&
                     this.standbyTasks.equals(other.standbyTasks) &&
                     this.partitionsByHost.equals(other.partitionsByHost);
@@ -189,7 +253,7 @@ public boolean equals(Object o) {
 
     @Override
     public String toString() {
-        return "[version=" + version + ", active tasks=" + activeTasks.size() + ", standby tasks=" + standbyTasks.size() + "]";
+        return "[version=" + usedVersion + ", active tasks=" + activeTasks.size() + ", standby tasks=" + standbyTasks.size() + "]";
     }
 
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
index f583dbafc94f1..7fee90b540213 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfo.java
@@ -31,42 +31,96 @@ public class SubscriptionInfo {
 
     private static final Logger log = LoggerFactory.getLogger(SubscriptionInfo.class);
 
-    private static final int CURRENT_VERSION = 2;
+    public static final int LATEST_SUPPORTED_VERSION = 2;
 
-    public final int version;
-    public final UUID processId;
-    public final Set prevTasks;
-    public final Set standbyTasks;
-    public final String userEndPoint;
+    private final int usedVersion;
+    private UUID processId;
+    private Set prevTasks;
+    private Set standbyTasks;
+    private String userEndPoint;
 
-    public SubscriptionInfo(UUID processId, Set prevTasks, Set standbyTasks, String userEndPoint) {
-        this(CURRENT_VERSION, processId, prevTasks, standbyTasks, userEndPoint);
+    private SubscriptionInfo(final int version) {
+        this.usedVersion = version;
     }
 
-    private SubscriptionInfo(int version, UUID processId, Set prevTasks, Set standbyTasks, String userEndPoint) {
-        this.version = version;
+    public SubscriptionInfo(final UUID processId,
+                            final Set prevTasks,
+                            final Set standbyTasks,
+                            final String userEndPoint) {
+        this(LATEST_SUPPORTED_VERSION, processId, prevTasks, standbyTasks, userEndPoint);
+    }
+
+    public SubscriptionInfo(final int version,
+                            final UUID processId,
+                            final Set prevTasks,
+                            final Set standbyTasks,
+                            final String userEndPoint) {
+        this.usedVersion = version;
         this.processId = processId;
         this.prevTasks = prevTasks;
         this.standbyTasks = standbyTasks;
         this.userEndPoint = userEndPoint;
     }
 
+    public int version() {
+        return usedVersion;
+    }
+
+    public UUID processId() {
+        return processId;
+    }
+
+    public Set prevTasks() {
+        return prevTasks;
+    }
+
+    public Set standbyTasks() {
+        return standbyTasks;
+    }
+
+    public String userEndPoint() {
+        return userEndPoint;
+    }
+
     /**
      * @throws TaskAssignmentException if method fails to encode the data
      */
     public ByteBuffer encode() {
-        byte[] endPointBytes;
-        if (userEndPoint == null) {
-            endPointBytes = new byte[0];
-        } else {
-            endPointBytes = userEndPoint.getBytes(Charset.forName("UTF-8"));
+        final ByteBuffer buf;
+
+        switch (usedVersion) {
+            case 1:
+                buf = encodeVersionOne();
+                break;
+            case 2:
+                buf = encodeVersionTwo(prepareUserEndPoint());
+                break;
+            default:
+                throw new IllegalStateException("Unknown metadata version: " + usedVersion
+                    + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
         }
-        ByteBuffer buf = ByteBuffer.allocate(4 /* version */ + 16 /* process id */ + 4 +
-                prevTasks.size() * 8 + 4 + standbyTasks.size() * 8
-                + 4 /* length of bytes */ + endPointBytes.length
-        );
-        // version
-        buf.putInt(version);
+
+        buf.rewind();
+        return buf;
+    }
+
+    private ByteBuffer encodeVersionOne() {
+        final ByteBuffer buf = ByteBuffer.allocate(getVersionOneByteLength());
+
+        buf.putInt(1); // version
+        encodeVersionOneData(buf);
+
+        return buf;
+    }
+
+    private int getVersionOneByteLength() {
+        return 4 + // version
+               16 + // client ID
+               4 + prevTasks.size() * 8 + // length + prev tasks
+               4 + standbyTasks.size() * 8; // length + standby tasks
+    }
+
+    private void encodeVersionOneData(final ByteBuffer buf) {
         // encode client UUID
         buf.putLong(processId.getMostSignificantBits());
         buf.putLong(processId.getLeastSignificantBits());
@@ -80,60 +134,104 @@ public ByteBuffer encode() {
         for (TaskId id : standbyTasks) {
             id.writeTo(buf);
         }
-        buf.putInt(endPointBytes.length);
-        buf.put(endPointBytes);
-        buf.rewind();
+    }
+
+    private byte[] prepareUserEndPoint() {
+        if (userEndPoint == null) {
+            return new byte[0];
+        } else {
+            return userEndPoint.getBytes(Charset.forName("UTF-8"));
+        }
+    }
+
+    private ByteBuffer encodeVersionTwo(final byte[] endPointBytes) {
+        final ByteBuffer buf = ByteBuffer.allocate(getVersionTwoByteLength(endPointBytes));
+
+        buf.putInt(2); // version
+        encodeVersionTwoData(buf, endPointBytes);
+
         return buf;
     }
 
+    private int getVersionTwoByteLength(final byte[] endPointBytes) {
+        return getVersionOneByteLength() +
+               4 + endPointBytes.length; // length + userEndPoint
+    }
+
+    private void encodeVersionTwoData(final ByteBuffer buf,
+                                      final byte[] endPointBytes) {
+        encodeVersionOneData(buf);
+        if (endPointBytes != null) {
+            buf.putInt(endPointBytes.length);
+            buf.put(endPointBytes);
+        }
+    }
+
     /**
      * @throws TaskAssignmentException if method fails to decode the data
      */
-    public static SubscriptionInfo decode(ByteBuffer data) {
+    public static SubscriptionInfo decode(final ByteBuffer data) {
         // ensure we are at the beginning of the ByteBuffer
         data.rewind();
 
-        // Decode version
-        int version = data.getInt();
-        if (version == CURRENT_VERSION || version == 1) {
-            // Decode client UUID
-            UUID processId = new UUID(data.getLong(), data.getLong());
-            // Decode previously active tasks
-            Set prevTasks = new HashSet<>();
-            int numPrevs = data.getInt();
-            for (int i = 0; i < numPrevs; i++) {
-                TaskId id = TaskId.readFrom(data);
-                prevTasks.add(id);
-            }
-            // Decode previously cached tasks
-            Set standbyTasks = new HashSet<>();
-            int numCached = data.getInt();
-            for (int i = 0; i < numCached; i++) {
-                standbyTasks.add(TaskId.readFrom(data));
-            }
-
-            String userEndPoint = null;
-            if (version == CURRENT_VERSION) {
-                int bytesLength = data.getInt();
-                if (bytesLength != 0) {
-                    byte[] bytes = new byte[bytesLength];
-                    data.get(bytes);
-                    userEndPoint = new String(bytes, Charset.forName("UTF-8"));
-                }
-
-            }
-            return new SubscriptionInfo(version, processId, prevTasks, standbyTasks, userEndPoint);
+        // decode used version
+        final int usedVersion = data.getInt();
+        final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(usedVersion);
+
+        switch (usedVersion) {
+            case 1:
+                decodeVersionOneData(subscriptionInfo, data);
+                break;
+            case 2:
+                decodeVersionTwoData(subscriptionInfo, data);
+                break;
+            default:
+                TaskAssignmentException fatalException = new TaskAssignmentException("Unable to decode subscription data: " +
+                    "used version: " + usedVersion + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
+                log.error(fatalException.getMessage(), fatalException);
+                throw fatalException;
+        }
+
+        return subscriptionInfo;
+    }
 
-        } else {
-            TaskAssignmentException ex = new TaskAssignmentException("unable to decode subscription data: version=" + version);
-            log.error(ex.getMessage(), ex);
-            throw ex;
+    private static void decodeVersionOneData(final SubscriptionInfo subscriptionInfo,
+                                             final ByteBuffer data) {
+        // decode client UUID
+        subscriptionInfo.processId = new UUID(data.getLong(), data.getLong());
+
+        // decode previously active tasks
+        final int numPrevs = data.getInt();
+        subscriptionInfo.prevTasks = new HashSet<>();
+        for (int i = 0; i < numPrevs; i++) {
+            TaskId id = TaskId.readFrom(data);
+            subscriptionInfo.prevTasks.add(id);
+        }
+
+        // decode previously cached tasks
+        final int numCached = data.getInt();
+        subscriptionInfo.standbyTasks = new HashSet<>();
+        for (int i = 0; i < numCached; i++) {
+            subscriptionInfo.standbyTasks.add(TaskId.readFrom(data));
+        }
+    }
+
+    private static void decodeVersionTwoData(final SubscriptionInfo subscriptionInfo,
+                                             final ByteBuffer data) {
+        decodeVersionOneData(subscriptionInfo, data);
+
+        // decode user end point (can be null)
+        int bytesLength = data.getInt();
+        if (bytesLength != 0) {
+            final byte[] bytes = new byte[bytesLength];
+            data.get(bytes);
+            subscriptionInfo.userEndPoint = new String(bytes, Charset.forName("UTF-8"));
         }
     }
 
     @Override
     public int hashCode() {
-        int hashCode = version ^ processId.hashCode() ^ prevTasks.hashCode() ^ standbyTasks.hashCode();
+        final int hashCode = usedVersion ^ processId.hashCode() ^ prevTasks.hashCode() ^ standbyTasks.hashCode();
         if (userEndPoint == null) {
             return hashCode;
         }
@@ -141,10 +239,10 @@ public int hashCode() {
     }
 
     @Override
-    public boolean equals(Object o) {
+    public boolean equals(final Object o) {
         if (o instanceof SubscriptionInfo) {
-            SubscriptionInfo other = (SubscriptionInfo) o;
-            return this.version == other.version &&
+            final SubscriptionInfo other = (SubscriptionInfo) o;
+            return this.usedVersion == other.usedVersion &&
                     this.processId.equals(other.processId) &&
                     this.prevTasks.equals(other.prevTasks) &&
                     this.standbyTasks.equals(other.standbyTasks) &&
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
index 8b4e8957ed59b..bb06c72d08060 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/QueryableStateIntegrationTest.java
@@ -376,7 +376,6 @@ public boolean conditionMet() {
         }
     }
 
-
     @Test
     public void queryOnRebalance() throws InterruptedException {
         final int numThreads = STREAM_TWO_PARTITIONS;
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
index bf3f1d1ac5ee3..b0c0d68287b19 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java
@@ -239,17 +239,17 @@ public void testAssignBasic() throws Exception {
 
         // the first consumer
         AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10"));
-        allActiveTasks.addAll(info10.activeTasks);
+        allActiveTasks.addAll(info10.activeTasks());
 
         // the second consumer
         AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11"));
-        allActiveTasks.addAll(info11.activeTasks);
+        allActiveTasks.addAll(info11.activeTasks());
 
         assertEquals(Utils.mkSet(task0, task1), allActiveTasks);
 
         // the third consumer
         AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
-        allActiveTasks.addAll(info20.activeTasks);
+        allActiveTasks.addAll(info20.activeTasks());
 
         assertEquals(3, allActiveTasks.size());
         assertEquals(allTasks, new HashSet<>(allActiveTasks));
@@ -317,13 +317,13 @@ public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() throws E
         final AssignmentInfo info10 = AssignmentInfo.decode(assignments.get("consumer10").userData());
 
         final List expectedInfo10TaskIds = Arrays.asList(taskIdA1, taskIdA3, taskIdB1, taskIdB3);
-        assertEquals(expectedInfo10TaskIds, info10.activeTasks);
+        assertEquals(expectedInfo10TaskIds, info10.activeTasks());
 
         // the second consumer
         final AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
         final List expectedInfo11TaskIds = Arrays.asList(taskIdA0, taskIdA2, taskIdB0, taskIdB2);
 
-        assertEquals(expectedInfo11TaskIds, info11.activeTasks);
+        assertEquals(expectedInfo11TaskIds, info11.activeTasks());
     }
 
     @Test
@@ -354,7 +354,7 @@ public void testAssignWithPartialTopology() throws Exception {
         // check assignment info
         Set allActiveTasks = new HashSet<>();
         AssignmentInfo info10 = checkAssignment(Utils.mkSet("topic1"), assignments.get("consumer10"));
-        allActiveTasks.addAll(info10.activeTasks);
+        allActiveTasks.addAll(info10.activeTasks());
 
         assertEquals(3, allActiveTasks.size());
         assertEquals(allTasks, new HashSet<>(allActiveTasks));
@@ -394,7 +394,7 @@ public void testAssignEmptyMetadata() throws Exception {
         // check assignment info
         Set allActiveTasks = new HashSet<>();
         AssignmentInfo info10 = checkAssignment(Collections.emptySet(), assignments.get("consumer10"));
-        allActiveTasks.addAll(info10.activeTasks);
+        allActiveTasks.addAll(info10.activeTasks());
 
         assertEquals(0, allActiveTasks.size());
         assertEquals(Collections.emptySet(), new HashSet<>(allActiveTasks));
@@ -407,7 +407,7 @@ public void testAssignEmptyMetadata() throws Exception {
 
         // the first consumer
         info10 = checkAssignment(allTopics, assignments.get("consumer10"));
-        allActiveTasks.addAll(info10.activeTasks);
+        allActiveTasks.addAll(info10.activeTasks());
 
         assertEquals(3, allActiveTasks.size());
         assertEquals(allTasks, new HashSet<>(allActiveTasks));
@@ -455,15 +455,15 @@ public void testAssignWithNewTasks() throws Exception {
         AssignmentInfo info;
 
         info = AssignmentInfo.decode(assignments.get("consumer10").userData());
-        allActiveTasks.addAll(info.activeTasks);
+        allActiveTasks.addAll(info.activeTasks());
         allPartitions.addAll(assignments.get("consumer10").partitions());
 
         info = AssignmentInfo.decode(assignments.get("consumer11").userData());
-        allActiveTasks.addAll(info.activeTasks);
+        allActiveTasks.addAll(info.activeTasks());
         allPartitions.addAll(assignments.get("consumer11").partitions());
 
         info = AssignmentInfo.decode(assignments.get("consumer20").userData());
-        allActiveTasks.addAll(info.activeTasks);
+        allActiveTasks.addAll(info.activeTasks());
         allPartitions.addAll(assignments.get("consumer20").partitions());
 
         assertEquals(allTasks, allActiveTasks);
@@ -524,14 +524,14 @@ public void testAssignWithStates() throws Exception {
         AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
         AssignmentInfo info20 = AssignmentInfo.decode(assignments.get("consumer20").userData());
 
-        assertEquals(2, info10.activeTasks.size());
-        assertEquals(2, info11.activeTasks.size());
-        assertEquals(2, info20.activeTasks.size());
+        assertEquals(2, info10.activeTasks().size());
+        assertEquals(2, info11.activeTasks().size());
+        assertEquals(2, info20.activeTasks().size());
 
         Set allTasks = new HashSet<>();
-        allTasks.addAll(info10.activeTasks);
-        allTasks.addAll(info11.activeTasks);
-        allTasks.addAll(info20.activeTasks);
+        allTasks.addAll(info10.activeTasks());
+        allTasks.addAll(info11.activeTasks());
+        allTasks.addAll(info20.activeTasks());
         assertEquals(new HashSet<>(tasks), allTasks);
 
         // check tasks for state topics
@@ -603,15 +603,15 @@ public void testAssignWithStandbyReplicas() throws Exception {
 
         // the first consumer
         AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10"));
-        allActiveTasks.addAll(info10.activeTasks);
-        allStandbyTasks.addAll(info10.standbyTasks.keySet());
+        allActiveTasks.addAll(info10.activeTasks());
+        allStandbyTasks.addAll(info10.standbyTasks().keySet());
 
         // the second consumer
         AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11"));
-        allActiveTasks.addAll(info11.activeTasks);
-        allStandbyTasks.addAll(info11.standbyTasks.keySet());
+        allActiveTasks.addAll(info11.activeTasks());
+        allStandbyTasks.addAll(info11.standbyTasks().keySet());
 
-        assertNotEquals("same processId has same set of standby tasks", info11.standbyTasks.keySet(), info10.standbyTasks.keySet());
+        assertNotEquals("same processId has same set of standby tasks", info11.standbyTasks().keySet(), info10.standbyTasks().keySet());
 
         // check active tasks assigned to the first client
         assertEquals(Utils.mkSet(task0, task1), new HashSet<>(allActiveTasks));
@@ -619,8 +619,8 @@ public void testAssignWithStandbyReplicas() throws Exception {
 
         // the third consumer
         AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
-        allActiveTasks.addAll(info20.activeTasks);
-        allStandbyTasks.addAll(info20.standbyTasks.keySet());
+        allActiveTasks.addAll(info20.activeTasks());
+        allStandbyTasks.addAll(info20.standbyTasks().keySet());
 
         // all task ids are in the active tasks and also in the standby tasks
 
@@ -847,7 +847,7 @@ public void shouldAddUserDefinedEndPointToSubscription() throws Exception {
         configurePartitionAssignor(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, (Object) userEndPoint));
         final PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("input"));
         final SubscriptionInfo subscriptionInfo = SubscriptionInfo.decode(subscription.userData());
-        assertEquals("localhost:8080", subscriptionInfo.userEndPoint);
+        assertEquals("localhost:8080", subscriptionInfo.userEndPoint());
     }
 
     @Test
@@ -874,7 +874,7 @@ public void shouldMapUserEndPointToTopicPartitions() throws Exception {
         final Map assignments = partitionAssignor.assign(metadata, subscriptions);
         final PartitionAssignor.Assignment consumerAssignment = assignments.get("consumer1");
         final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumerAssignment.userData());
-        final Set topicPartitions = assignmentInfo.partitionsByHost.get(new HostInfo("localhost", 8080));
+        final Set topicPartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
         assertEquals(Utils.mkSet(new TopicPartition("topic1", 0),
                 new TopicPartition("topic1", 1),
                 new TopicPartition("topic1", 2)), topicPartitions);
@@ -1072,8 +1072,8 @@ public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() throws Except
         final Map assign = partitionAssignor.assign(metadata, subscriptions);
         final PartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
         final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
-        final Set consumer1partitions = assignmentInfo.partitionsByHost.get(new HostInfo("localhost", 8080));
-        final Set consumer2Partitions = assignmentInfo.partitionsByHost.get(new HostInfo("other", 9090));
+        final Set consumer1partitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
+        final Set consumer2Partitions = assignmentInfo.partitionsByHost().get(new HostInfo("other", 9090));
         final HashSet allAssignedPartitions = new HashSet<>(consumer1partitions);
         allAssignedPartitions.addAll(consumer2Partitions);
         assertThat(consumer1partitions, not(allPartitions));
@@ -1095,6 +1095,37 @@ public void shouldThrowKafkaExceptionIfStreamThreadConfigIsNotThreadDataProvider
         partitionAssignor.configure(config);
     }
 
+    @Test
+    public void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions() throws Exception {
+        final Map subscriptions = new HashMap<>();
+        final Set emptyTasks = Collections.emptySet();
+        subscriptions.put(
+            "consumer1",
+            new PartitionAssignor.Subscription(
+                Collections.singletonList("topic1"),
+                new SubscriptionInfo(1, UUID.randomUUID(), emptyTasks, emptyTasks, null).encode()
+            )
+        );
+        subscriptions.put(
+            "consumer2",
+            new PartitionAssignor.Subscription(
+                Collections.singletonList("topic1"),
+                new SubscriptionInfo(2, UUID.randomUUID(), emptyTasks, emptyTasks, null).encode()
+            )
+        );
+
+        mockTaskManager(Collections.emptySet(),
+            Collections.emptySet(),
+            UUID.randomUUID(),
+            new InternalTopologyBuilder());
+        partitionAssignor.configure(configProps());
+        final Map assignment = partitionAssignor.assign(metadata, subscriptions);
+
+        assertThat(assignment.size(), equalTo(2));
+        assertThat(AssignmentInfo.decode(assignment.get("consumer1").userData()).version(), equalTo(1));
+        assertThat(AssignmentInfo.decode(assignment.get("consumer2").userData()).version(), equalTo(1));
+    }
+
     private PartitionAssignor.Assignment createAssignment(final Map> firstHostState) {
         final AssignmentInfo info = new AssignmentInfo(Collections.emptyList(),
                                                        Collections.>emptyMap(),
@@ -1111,7 +1142,7 @@ private AssignmentInfo checkAssignment(Set expectedTopics, PartitionAssi
         AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
 
         // check if the number of assigned partitions == the size of active task id list
-        assertEquals(assignment.partitions().size(), info.activeTasks.size());
+        assertEquals(assignment.partitions().size(), info.activeTasks().size());
 
         // check if active tasks are consistent
         List activeTasks = new ArrayList<>();
@@ -1121,14 +1152,14 @@ private AssignmentInfo checkAssignment(Set expectedTopics, PartitionAssi
             activeTasks.add(new TaskId(0, partition.partition()));
             activeTopics.add(partition.topic());
         }
-        assertEquals(activeTasks, info.activeTasks);
+        assertEquals(activeTasks, info.activeTasks());
 
         // check if active partitions cover all topics
         assertEquals(expectedTopics, activeTopics);
 
         // check if standby tasks are consistent
         Set standbyTopics = new HashSet<>();
-        for (Map.Entry> entry : info.standbyTasks.entrySet()) {
+        for (Map.Entry> entry : info.standbyTasks().entrySet()) {
             TaskId id = entry.getKey();
             Set partitions = entry.getValue();
             for (TopicPartition partition : partitions) {
@@ -1139,7 +1170,7 @@ private AssignmentInfo checkAssignment(Set expectedTopics, PartitionAssi
             }
         }
 
-        if (info.standbyTasks.size() > 0) {
+        if (info.standbyTasks().size() > 0) {
             // check if standby partitions cover all topics
             assertEquals(expectedTopics, standbyTopics);
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
index ec94ad81acd6d..726a5623cd560 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java
@@ -33,6 +33,7 @@
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 public class AssignmentInfoTest {
 
@@ -61,10 +62,10 @@ public void shouldDecodePreviousVersion() throws IOException {
         standbyTasks.put(new TaskId(2, 0), Utils.mkSet(new TopicPartition("t3", 0), new TopicPartition("t3", 0)));
         final AssignmentInfo oldVersion = new AssignmentInfo(1, activeTasks, standbyTasks, null);
         final AssignmentInfo decoded = AssignmentInfo.decode(encodeV1(oldVersion));
-        assertEquals(oldVersion.activeTasks, decoded.activeTasks);
-        assertEquals(oldVersion.standbyTasks, decoded.standbyTasks);
-        assertEquals(0, decoded.partitionsByHost.size()); // should be empty as wasn't in V1
-        assertEquals(2, decoded.version); // automatically upgraded to v2 on decode;
+        assertEquals(oldVersion.activeTasks(), decoded.activeTasks());
+        assertEquals(oldVersion.standbyTasks(), decoded.standbyTasks());
+        assertNull(decoded.partitionsByHost()); // should be null as wasn't in V1
+        assertEquals(1, decoded.version());
     }
 
 
@@ -76,15 +77,15 @@ private ByteBuffer encodeV1(AssignmentInfo oldVersion) throws IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream out = new DataOutputStream(baos);
         // Encode version
-        out.writeInt(oldVersion.version);
+        out.writeInt(oldVersion.version());
         // Encode active tasks
-        out.writeInt(oldVersion.activeTasks.size());
-        for (TaskId id : oldVersion.activeTasks) {
+        out.writeInt(oldVersion.activeTasks().size());
+        for (TaskId id : oldVersion.activeTasks()) {
             id.writeTo(out);
         }
         // Encode standby tasks
-        out.writeInt(oldVersion.standbyTasks.size());
-        for (Map.Entry> entry : oldVersion.standbyTasks.entrySet()) {
+        out.writeInt(oldVersion.standbyTasks().size());
+        for (Map.Entry> entry : oldVersion.standbyTasks().entrySet()) {
             TaskId id = entry.getKey();
             id.writeTo(out);
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
index 9c011bb0caeb9..633285a2b4ddc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/SubscriptionInfoTest.java
@@ -65,14 +65,12 @@ public void shouldBeBackwardCompatible() {
 
         final ByteBuffer v1Encoding = encodePreviousVersion(processId, activeTasks, standbyTasks);
         final SubscriptionInfo decode = SubscriptionInfo.decode(v1Encoding);
-        assertEquals(activeTasks, decode.prevTasks);
-        assertEquals(standbyTasks, decode.standbyTasks);
-        assertEquals(processId, decode.processId);
-        assertNull(decode.userEndPoint);
-
+        assertEquals(activeTasks, decode.prevTasks());
+        assertEquals(standbyTasks, decode.standbyTasks());
+        assertEquals(processId, decode.processId());
+        assertNull(decode.userEndPoint());
     }
 
-
     /**
      * This is a clone of what the V1 encoding did. The encode method has changed for V2
      * so it is impossible to test compatibility without having this

From 8a7d7e7955889125b1196caeded6fa57d93a0b46 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Mon, 5 Mar 2018 14:06:32 -0500
Subject: [PATCH 0113/1847] MINOR: Add System test for standby task-rebalancing
 (#4554)

Author: Bill Bejeck 

Reviewers: Damian Guy , Guozhang Wang , Matthias J. Sax 
---
 .../StreamsRepeatingIntegerKeyProducer.java   | 108 ++++++++++++
 .../tests/StreamsStandByReplicaTest.java      | 162 +++++++++++++++++
 .../kafka/streams/tests/SystemTestUtil.java   |  62 +++++++
 .../streams/tests/SystemTestUtilTest.java     |  75 ++++++++
 tests/kafkatest/services/streams.py           |  17 +-
 .../streams/streams_standby_replica_test.py   | 166 ++++++++++++++++++
 6 files changed, 589 insertions(+), 1 deletion(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/tests/StreamsRepeatingIntegerKeyProducer.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/tests/StreamsStandByReplicaTest.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtil.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtilTest.java
 create mode 100644 tests/kafkatest/tests/streams/streams_standby_replica_test.py

diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsRepeatingIntegerKeyProducer.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsRepeatingIntegerKeyProducer.java
new file mode 100644
index 0000000000000..85ca077cca033
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsRepeatingIntegerKeyProducer.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Utils;
+
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Utility class used to send messages with integer keys
+ * repeating in sequence every 1000 messages.  Multiple topics for publishing
+ * can be provided in the config map with key of 'topics' and ';' delimited list of output topics
+ */
+public class StreamsRepeatingIntegerKeyProducer {
+
+    private static volatile boolean keepProducing = true;
+    private volatile static int messageCounter = 0;
+
+    public static void main(String[] args) {
+        System.out.println("StreamsTest instance started");
+
+        final String kafka = args.length > 0 ? args[0] : "localhost:9092";
+        final String configString = args.length > 2 ? args[2] : null;
+
+        final Map configs = SystemTestUtil.parseConfigs(configString);
+        System.out.println("Using provided configs " + configs);
+
+        final int numMessages = configs.containsKey("num_messages") ? Integer.parseInt(configs.get("num_messages")) : 1000;
+
+        final Properties producerProps = new Properties();
+        producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "StreamsRepeatingIntegerKeyProducer");
+        producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
+        producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
+        producerProps.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
+
+        final String value = "testingValue";
+        Integer key = 0;
+
+        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+            @Override
+            public void run() {
+                keepProducing = false;
+            }
+        }));
+
+        final String[] topics = configs.get("topics").split(";");
+        final int totalMessagesToProduce = numMessages * topics.length;
+
+        try (final KafkaProducer kafkaProducer = new KafkaProducer<>(producerProps)) {
+
+            while (keepProducing && messageCounter < totalMessagesToProduce) {
+                for (final String topic : topics) {
+                    final ProducerRecord producerRecord = new ProducerRecord<>(topic, key.toString(), value + key);
+                    kafkaProducer.send(producerRecord, new Callback() {
+                        @Override
+                        public void onCompletion(final RecordMetadata metadata, final Exception exception) {
+                            if (exception != null) {
+                                exception.printStackTrace(System.err);
+                                System.err.flush();
+                                if (exception instanceof TimeoutException) {
+                                    try {
+                                        // message == org.apache.kafka.common.errors.TimeoutException: Expiring 4 record(s) for data-0: 30004 ms has passed since last attempt plus backoff time
+                                        final int expired = Integer.parseInt(exception.getMessage().split(" ")[2]);
+                                        messageCounter -= expired;
+                                    } catch (final Exception ignore) {
+                                    }
+                                }
+                            }
+                        }
+                    });
+                    messageCounter += 1;
+                }
+                key += 1;
+                if (key % 1000 == 0) {
+                    System.out.println("Sent 1000 messages");
+                    Utils.sleep(100);
+                    key = 0;
+                }
+            }
+        }
+        System.out.println("Producer shut down now, sent total " + messageCounter  + " of requested " + totalMessagesToProduce);
+        System.out.flush();
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsStandByReplicaTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsStandByReplicaTest.java
new file mode 100644
index 0000000000000..0e3aa13bc9d01
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsStandByReplicaTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.tests;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.Consumed;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.Materialized;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.streams.kstream.ValueMapper;
+import org.apache.kafka.streams.processor.ThreadMetadata;
+import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier;
+import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.test.TestUtils;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public class StreamsStandByReplicaTest {
+
+    public static void main(String[] args) {
+
+        System.out.println("StreamsTest instance started");
+
+        final String kafka = args.length > 0 ? args[0] : "localhost:9092";
+        final String stateDirStr = args.length > 1 ? args[1] : TestUtils.tempDirectory().getAbsolutePath();
+        final String additionalConfigs = args.length > 2 ? args[2] : null;
+
+        final Serde stringSerde = Serdes.String();
+
+        final Properties streamsProperties = new Properties();
+        streamsProperties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
+        streamsProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "kafka-streams-standby-tasks");
+        streamsProperties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        streamsProperties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        streamsProperties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
+        streamsProperties.put(StreamsConfig.STATE_DIR_CONFIG, stateDirStr);
+        streamsProperties.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
+        streamsProperties.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
+        streamsProperties.put(StreamsConfig.producerPrefix(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG), true);
+
+        final Map updated = SystemTestUtil.parseConfigs(additionalConfigs);
+        System.out.println("Updating configs with " + updated);
+
+        final String sourceTopic = updated.remove("sourceTopic");
+        final String sinkTopic1 = updated.remove("sinkTopic1");
+        final String sinkTopic2 = updated.remove("sinkTopic2");
+
+        if (sourceTopic == null || sinkTopic1 == null || sinkTopic2 == null) {
+            System.err.println(String.format("one or more required topics null sourceTopic[%s], sinkTopic1[%s], sinkTopic2[%s]", sourceTopic, sinkTopic1, sinkTopic2));
+            System.err.flush();
+            System.exit(1);
+        }
+
+        streamsProperties.putAll(updated);
+
+        if (!confirmCorrectConfigs(streamsProperties)) {
+            System.err.println(String.format("ERROR: Did not have all required configs expected  to contain %s, %s,  %s,  %s",
+                                             StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG),
+                                             StreamsConfig.producerPrefix(ProducerConfig.RETRIES_CONFIG),
+                                             StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG),
+                                             StreamsConfig.producerPrefix(ProducerConfig.MAX_BLOCK_MS_CONFIG)));
+
+            System.exit(1);
+        }
+
+        final StreamsBuilder builder = new StreamsBuilder();
+
+        String inMemoryStoreName = "in-memory-store";
+        String persistentMemoryStoreName = "persistent-memory-store";
+
+        KeyValueBytesStoreSupplier inMemoryStoreSupplier = Stores.inMemoryKeyValueStore(inMemoryStoreName);
+        KeyValueBytesStoreSupplier persistentStoreSupplier = Stores.persistentKeyValueStore(persistentMemoryStoreName);
+
+        KStream inputStream = builder.stream(sourceTopic, Consumed.with(stringSerde, stringSerde));
+
+        ValueMapper countMapper = new ValueMapper() {
+            @Override
+            public String apply(final Long value) {
+                return value.toString();
+            }
+        };
+
+        inputStream.groupByKey().count(Materialized.as(inMemoryStoreSupplier)).toStream().mapValues(countMapper)
+            .to(sinkTopic1, Produced.with(stringSerde, stringSerde));
+
+        inputStream.groupByKey().count(Materialized.as(persistentStoreSupplier)).toStream().mapValues(countMapper)
+            .to(sinkTopic2, Produced.with(stringSerde, stringSerde));
+
+        final KafkaStreams streams = new KafkaStreams(builder.build(), streamsProperties);
+
+        streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+            @Override
+            public void uncaughtException(final Thread t, final Throwable e) {
+                System.err.println("FATAL: An unexpected exception " + e);
+                e.printStackTrace(System.err);
+                System.err.flush();
+                shutdown(streams);
+            }
+        });
+
+        streams.setStateListener(new KafkaStreams.StateListener() {
+            @Override
+            public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
+                if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
+                    final Set threadMetadata = streams.localThreadsMetadata();
+                    for (final ThreadMetadata threadMetadatum : threadMetadata) {
+                        System.out.println("ACTIVE_TASKS:" + threadMetadatum.activeTasks().size() + " STANDBY_TASKS:" + threadMetadatum.standbyTasks().size());
+                    }
+                }
+            }
+        });
+
+        System.out.println("Start Kafka Streams");
+        streams.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+            @Override
+            public void run() {
+                shutdown(streams);
+                System.out.println("Shut down streams now");
+            }
+        }));
+
+
+    }
+
+    private static void shutdown(final KafkaStreams streams) {
+        streams.close(10, TimeUnit.SECONDS);
+    }
+
+    private static boolean confirmCorrectConfigs(final Properties properties) {
+        return properties.containsKey(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG)) &&
+               properties.containsKey(StreamsConfig.producerPrefix(ProducerConfig.RETRIES_CONFIG)) &&
+               properties.containsKey(StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG)) &&
+               properties.containsKey(StreamsConfig.producerPrefix(ProducerConfig.MAX_BLOCK_MS_CONFIG));
+    }
+
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtil.java b/streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtil.java
new file mode 100644
index 0000000000000..4ddbf693dc931
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtil.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.tests;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Class for common convenience methods for working on
+ * System tests
+ */
+
+public class SystemTestUtil {
+
+    private static final int KEY = 0;
+    private static final int VALUE = 1;
+
+    /**
+     * Takes a string with keys and values separated by '=' and each key value pair
+     * separated by ',' for example max.block.ms=5000,retries=6,request.timeout.ms=6000
+     *
+     * This class makes it easier to pass configs from the system test in python to the Java test.
+     *
+     * @param formattedConfigs the formatted config string
+     * @return HashMap with keys and values inserted
+     */
+    public static Map parseConfigs(final String formattedConfigs) {
+        Objects.requireNonNull(formattedConfigs, "Formatted config String can't be null");
+
+        if (formattedConfigs.indexOf('=') == -1) {
+            throw new IllegalStateException(String.format("Provided string [ %s ] does not have expected key-value separator of '='", formattedConfigs));
+        }
+
+        final String[] parts = formattedConfigs.split(",");
+        final Map configs = new HashMap<>();
+        for (final String part : parts) {
+            final String[] keyValue = part.split("=");
+            if (keyValue.length > 2) {
+                throw new IllegalStateException(
+                    String.format("Provided string [ %s ] does not have expected key-value pair separator of ','", formattedConfigs));
+            }
+            configs.put(keyValue[KEY], keyValue[VALUE]);
+        }
+        return configs;
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtilTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtilTest.java
new file mode 100644
index 0000000000000..f55450d2c955b
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/SystemTestUtilTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.tests;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.junit.Assert.assertEquals;
+
+public class SystemTestUtilTest {
+
+    private final Map expectedParsedMap = new TreeMap<>();
+
+    @Before
+    public void setUp() {
+        expectedParsedMap.put("foo", "foo1");
+        expectedParsedMap.put("bar", "bar1");
+        expectedParsedMap.put("baz", "baz1");
+    }
+
+    @Test
+    public void shouldParseCorrectMap() {
+        final String formattedConfigs = "foo=foo1,bar=bar1,baz=baz1";
+        final Map parsedMap = SystemTestUtil.parseConfigs(formattedConfigs);
+        final TreeMap sortedParsedMap = new TreeMap<>(parsedMap);
+        assertEquals(sortedParsedMap, expectedParsedMap);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void shouldThrowExceptionOnNull() {
+        SystemTestUtil.parseConfigs(null);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void shouldThrowExceptionIfNotCorrectKeyValueSeparator() {
+        final String badString = "foo:bar,baz:boo";
+        SystemTestUtil.parseConfigs(badString);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void shouldThrowExceptionIfNotCorrectKeyValuePairSeparator() {
+        final String badString = "foo=bar;baz=boo";
+        SystemTestUtil.parseConfigs(badString);
+    }
+
+    @Test
+    public void shouldParseSingleKeyValuePairString() {
+        final Map expectedSinglePairMap = new HashMap<>();
+        expectedSinglePairMap.put("foo", "bar");
+        final String singleValueString = "foo=bar";
+        final Map parsedMap = SystemTestUtil.parseConfigs(singleValueString);
+        assertEquals(expectedSinglePairMap, parsedMap);
+    }
+
+
+}
\ No newline at end of file
diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py
index 9df87ca794594..e552a39eaa0e8 100644
--- a/tests/kafkatest/services/streams.py
+++ b/tests/kafkatest/services/streams.py
@@ -18,7 +18,6 @@
 
 from ducktape.services.service import Service
 from ducktape.utils.util import wait_until
-
 from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
 
 
@@ -236,3 +235,19 @@ def start_cmd(self, node):
               " %(user_test_args3)s & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args
 
         return cmd
+
+
+class StreamsStandbyTaskService(StreamsTestBaseService):
+    def __init__(self, test_context, kafka, configs):
+        super(StreamsStandbyTaskService, self).__init__(test_context,
+                                                        kafka,
+                                                        "org.apache.kafka.streams.tests.StreamsStandByReplicaTest",
+                                                        configs)
+
+
+class StreamsRepeatingIntegerKeyProducerService(StreamsTestBaseService):
+    def __init__(self, test_context, kafka, configs):
+        super(StreamsRepeatingIntegerKeyProducerService, self).__init__(test_context,
+                                                                        kafka,
+                                                                        "org.apache.kafka.streams.tests.StreamsRepeatingIntegerKeyProducer",
+                                                                        configs)
diff --git a/tests/kafkatest/tests/streams/streams_standby_replica_test.py b/tests/kafkatest/tests/streams/streams_standby_replica_test.py
new file mode 100644
index 0000000000000..533d4b5c1819a
--- /dev/null
+++ b/tests/kafkatest/tests/streams/streams_standby_replica_test.py
@@ -0,0 +1,166 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ducktape.tests.test import Test
+from ducktape.utils.util import wait_until
+from kafkatest.services.kafka import KafkaService
+from kafkatest.services.streams import StreamsRepeatingIntegerKeyProducerService
+from kafkatest.services.streams import StreamsStandbyTaskService
+from kafkatest.services.verifiable_consumer import VerifiableConsumer
+from kafkatest.services.zookeeper import ZookeeperService
+
+
+class StreamsStandbyTask(Test):
+    """
+    This test validates using standby tasks helps with rebalance times
+    additionally verifies standby replicas continue to work in the
+    face of continual changes to streams code base
+    """
+
+    streams_source_topic = "standbyTaskSource1"
+    streams_sink_topic_1 = "standbyTaskSink1"
+    streams_sink_topic_2 = "standbyTaskSink2"
+
+    num_messages = 60000
+
+    def __init__(self, test_context):
+        super(StreamsStandbyTask, self).__init__(test_context=test_context)
+        self.zk = ZookeeperService(test_context, num_nodes=1)
+        self.kafka = KafkaService(test_context,
+                                  num_nodes=3,
+                                  zk=self.zk,
+                                  topics={
+                                      self.streams_source_topic: {'partitions': 6, 'replication-factor': 1},
+                                      self.streams_sink_topic_1: {'partitions': 1, 'replication-factor': 1},
+                                      self.streams_sink_topic_2: {'partitions': 1, 'replication-factor': 1}
+                                  })
+
+    def get_consumer(self, topic, num_messages):
+        return VerifiableConsumer(self.test_context,
+                                  1,
+                                  self.kafka,
+                                  topic,
+                                  "stream-broker-resilience-verify-consumer",
+                                  max_messages=num_messages)
+
+    def assert_consume(self, test_state, topic, num_messages=5):
+        consumer = self.get_consumer(topic, num_messages)
+        consumer.start()
+
+        wait_until(lambda: consumer.total_consumed() >= num_messages,
+                   timeout_sec=120,
+                   err_msg="At %s streams did not process messages in 60 seconds " % test_state)
+
+    @staticmethod
+    def get_configs(extra_configs=""):
+        # Consumer max.poll.interval > min(max.block.ms, ((retries + 1) * request.timeout)
+        consumer_poll_ms = "consumer.max.poll.interval.ms=50000"
+        retries_config = "producer.retries=2"
+        request_timeout = "producer.request.timeout.ms=15000"
+        max_block_ms = "producer.max.block.ms=30000"
+
+        # java code expects configs in key=value,key=value format
+        updated_configs = consumer_poll_ms + "," + retries_config + "," + request_timeout + "," + max_block_ms + extra_configs
+
+        return updated_configs
+
+    def wait_for_verification(self, processor, message, file, num_lines=1, timeout_sec=20):
+        wait_until(lambda: self.verify_from_file(processor, message, file) >= num_lines,
+                   timeout_sec=timeout_sec,
+                   err_msg="Did expect to read '%s' from %s" % (message, processor.node.account))
+
+    @staticmethod
+    def verify_from_file(processor, message, file):
+        result = processor.node.account.ssh_output("grep '%s' %s | wc -l" % (message, file), allow_fail=False)
+        return int(result)
+
+
+    def setUp(self):
+        self.zk.start()
+        self.kafka.start()
+
+    def test_standby_tasks_rebalance(self):
+
+        driver_configs = "num_messages=%s,topics=%s" % (str(self.num_messages), self.streams_source_topic)
+
+        driver = StreamsRepeatingIntegerKeyProducerService(self.test_context, self.kafka, driver_configs)
+        driver.start()
+
+        configs = self.get_configs(",sourceTopic=%s,sinkTopic1=%s,sinkTopic2=%s" % (self.streams_source_topic,
+                                                                                    self.streams_sink_topic_1,
+                                                                                    self.streams_sink_topic_2))
+
+        processor_1 = StreamsStandbyTaskService(self.test_context, self.kafka, configs)
+        processor_2 = StreamsStandbyTaskService(self.test_context, self.kafka, configs)
+        processor_3 = StreamsStandbyTaskService(self.test_context, self.kafka, configs)
+
+        processor_1.start()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:6 STANDBY_TASKS:0", processor_1.STDOUT_FILE)
+
+        processor_2.start()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_1.STDOUT_FILE)
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_2.STDOUT_FILE)
+
+        processor_3.start()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_1.STDOUT_FILE)
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_2.STDOUT_FILE)
+        self.wait_for_verification(processor_3, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_3.STDOUT_FILE)
+
+        processor_1.stop()
+
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_2.STDOUT_FILE, num_lines=2)
+        self.wait_for_verification(processor_3, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_3.STDOUT_FILE)
+
+        processor_2.stop()
+
+        self.wait_for_verification(processor_3, "ACTIVE_TASKS:6 STANDBY_TASKS:0", processor_3.STDOUT_FILE)
+
+        processor_1.start()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_1.STDOUT_FILE)
+        self.wait_for_verification(processor_3, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_3.STDOUT_FILE, num_lines=2)
+
+        processor_2.start()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_1.STDOUT_FILE)
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_2.STDOUT_FILE)
+        self.wait_for_verification(processor_3, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_3.STDOUT_FILE, num_lines=2)
+
+        processor_3.stop()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_1.STDOUT_FILE, num_lines=2)
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:3 STANDBY_TASKS:3", processor_2.STDOUT_FILE)
+
+        processor_1.stop()
+
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:6 STANDBY_TASKS:0", processor_2.STDOUT_FILE)
+
+        processor_3.start()
+        processor_1.start()
+
+        self.wait_for_verification(processor_1, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_1.STDOUT_FILE)
+        self.wait_for_verification(processor_3, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_3.STDOUT_FILE)
+        self.wait_for_verification(processor_2, "ACTIVE_TASKS:2 STANDBY_TASKS:2", processor_2.STDOUT_FILE, num_lines=2)
+
+        self.assert_consume("assert all messages consumed from %s" % self.streams_sink_topic_1, self.streams_sink_topic_1, self.num_messages)
+        self.assert_consume("assert all messages consumed from %s" % self.streams_sink_topic_2, self.streams_sink_topic_2, self.num_messages)
+
+        self.wait_for_verification(driver, "Producer shut down now, sent total {0} of requested {0}".format(str(self.num_messages)),
+                                   driver.STDOUT_FILE)
+
+

From 8f2c08716630eba7e3badacc79be4c8c413a00da Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 5 Mar 2018 16:48:05 -0800
Subject: [PATCH 0114/1847] MINOR: Complete inflight requests in order on
 disconnect (#4642)

NetworkClient should use FIFO order when completing inflight requests following a disconnect.

I've added new unit tests for `InFlightRequests` and `NetworkClient` which verify completion order.

Reviewers: Jun Rao 
---
 .../kafka/clients/InFlightRequests.java       | 11 +++-
 .../kafka/clients/InFlightRequestsTest.java   | 63 +++++++++++++++----
 .../kafka/clients/NetworkClientTest.java      | 52 +++++++++++++++
 .../org/apache/kafka/test/MockSelector.java   | 52 +++++++++++----
 4 files changed, 151 insertions(+), 27 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
index a062818e3b2c4..5caee2d4c87d8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
+++ b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
@@ -20,12 +20,12 @@
 import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
-
 /**
  * The set of requests which have been sent or are being sent but haven't yet received a response
  */
@@ -151,9 +151,14 @@ public Iterable clearAll(String node) {
         if (reqs == null) {
             return Collections.emptyList();
         } else {
-            Deque clearedRequests = requests.remove(node);
+            final Deque clearedRequests = requests.remove(node);
             inFlightRequestCount.getAndAdd(-clearedRequests.size());
-            return clearedRequests;
+            return new Iterable() {
+                @Override
+                public Iterator iterator() {
+                    return clearedRequests.descendingIterator();
+                }
+            };
         }
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/InFlightRequestsTest.java b/clients/src/test/java/org/apache/kafka/clients/InFlightRequestsTest.java
index e00ca08610660..600e5dc9053d4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/InFlightRequestsTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/InFlightRequestsTest.java
@@ -17,44 +17,85 @@
 
 package org.apache.kafka.clients;
 
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.apache.kafka.test.TestUtils;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.List;
+
 import static org.junit.Assert.assertEquals;
 
 public class InFlightRequestsTest {
 
     private InFlightRequests inFlightRequests;
+    private int correlationId;
+    private String dest = "dest";
+
     @Before
     public void setup() {
         inFlightRequests = new InFlightRequests(12);
-        NetworkClient.InFlightRequest ifr =
-                new NetworkClient.InFlightRequest(null, 0, "dest", null, false, false, null, null, 0);
-        inFlightRequests.add(ifr);
+        correlationId = 0;
     }
 
     @Test
-    public void checkIncrementAndDecrementOnLastSent() {
+    public void testCompleteLastSent() {
+        int correlationId1 = addRequest(dest);
+        int correlationId2 = addRequest(dest);
+        assertEquals(2, inFlightRequests.count());
+
+        assertEquals(correlationId2, inFlightRequests.completeLastSent(dest).header.correlationId());
         assertEquals(1, inFlightRequests.count());
 
-        inFlightRequests.completeLastSent("dest");
+        assertEquals(correlationId1, inFlightRequests.completeLastSent(dest).header.correlationId());
         assertEquals(0, inFlightRequests.count());
     }
 
     @Test
-    public void checkDecrementOnClear() {
-        inFlightRequests.clearAll("dest");
+    public void testClearAll() {
+        int correlationId1 = addRequest(dest);
+        int correlationId2 = addRequest(dest);
+
+        List clearedRequests = TestUtils.toList(this.inFlightRequests.clearAll(dest));
         assertEquals(0, inFlightRequests.count());
+        assertEquals(2, clearedRequests.size());
+        assertEquals(correlationId1, clearedRequests.get(0).header.correlationId());
+        assertEquals(correlationId2, clearedRequests.get(1).header.correlationId());
     }
 
     @Test
-    public void checkDecrementOnCompleteNext() {
-        inFlightRequests.completeNext("dest");
+    public void testCompleteNext() {
+        int correlationId1 = addRequest(dest);
+        int correlationId2 = addRequest(dest);
+        assertEquals(2, inFlightRequests.count());
+
+        assertEquals(correlationId1, inFlightRequests.completeNext(dest).header.correlationId());
+        assertEquals(1, inFlightRequests.count());
+
+        assertEquals(correlationId2, inFlightRequests.completeNext(dest).header.correlationId());
         assertEquals(0, inFlightRequests.count());
     }
 
     @Test(expected = IllegalStateException.class)
-    public void throwExceptionOnNeverBeforeSeenNode() {
-        inFlightRequests.completeNext("not-added");
+    public void testCompleteNextThrowsIfNoInflights() {
+        inFlightRequests.completeNext(dest);
     }
+
+    @Test(expected = IllegalStateException.class)
+    public void testCompleteLastSentThrowsIfNoInFlights() {
+        inFlightRequests.completeLastSent(dest);
+    }
+
+    private int addRequest(String destination) {
+        int correlationId = this.correlationId;
+        this.correlationId += 1;
+
+        RequestHeader requestHeader = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", correlationId);
+        NetworkClient.InFlightRequest ifr = new NetworkClient.InFlightRequest(requestHeader, 0,
+                destination, null, false, false, null, null, 0);
+        inFlightRequests.add(ifr);
+        return correlationId;
+    }
+
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index 8c2428e449265..77b36df677b8a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -37,12 +37,14 @@
 import org.junit.Test;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
 public class NetworkClientTest {
@@ -82,6 +84,7 @@ private NetworkClient createNetworkClientWithNoVersionDiscovery() {
 
     @Before
     public void setup() {
+        selector.reset();
         metadata.update(cluster, Collections.emptySet(), time.milliseconds());
     }
 
@@ -350,6 +353,55 @@ public void testClientDisconnectAfterInternalApiVersionRequest() throws Exceptio
         assertTrue(responses.isEmpty());
     }
 
+    @Test
+    public void testDisconnectWithMultipleInFlights() throws Exception {
+        NetworkClient client = this.clientWithNoVersionDiscovery;
+        awaitReady(client, node);
+        assertTrue("Expected NetworkClient to be ready to send to node " + node.idString(),
+                client.isReady(node, time.milliseconds()));
+
+        MetadataRequest.Builder builder = new MetadataRequest.Builder(Collections.emptyList(), true);
+        long now = time.milliseconds();
+
+        final List callbackResponses = new ArrayList<>();
+        RequestCompletionHandler callback = new RequestCompletionHandler() {
+            @Override
+            public void onComplete(ClientResponse response) {
+                callbackResponses.add(response);
+            }
+        };
+
+        ClientRequest request1 = client.newClientRequest(node.idString(), builder, now, true, callback);
+        client.send(request1, now);
+        client.poll(0, now);
+
+        ClientRequest request2 = client.newClientRequest(node.idString(), builder, now, true, callback);
+        client.send(request2, now);
+        client.poll(0, now);
+
+        assertNotEquals(request1.correlationId(), request2.correlationId());
+
+        assertEquals(2, client.inFlightRequestCount());
+        assertEquals(2, client.inFlightRequestCount(node.idString()));
+
+        client.disconnect(node.idString());
+
+        List responses = client.poll(0, time.milliseconds());
+        assertEquals(2, responses.size());
+        assertEquals(responses, callbackResponses);
+        assertEquals(0, client.inFlightRequestCount());
+        assertEquals(0, client.inFlightRequestCount(node.idString()));
+
+        // Ensure that the responses are returned in the order they were sent
+        ClientResponse response1 = responses.get(0);
+        assertTrue(response1.wasDisconnected());
+        assertEquals(request1.correlationId(), response1.requestHeader().correlationId());
+
+        ClientResponse response2 = responses.get(1);
+        assertTrue(response2.wasDisconnected());
+        assertEquals(request2.correlationId(), response2.requestHeader().correlationId());
+    }
+
     @Test
     public void testCallDisconnect() throws Exception {
         awaitReady(client, node);
diff --git a/clients/src/test/java/org/apache/kafka/test/MockSelector.java b/clients/src/test/java/org/apache/kafka/test/MockSelector.java
index 6fc1b1b2acad8..bd27d5c6d9903 100644
--- a/clients/src/test/java/org/apache/kafka/test/MockSelector.java
+++ b/clients/src/test/java/org/apache/kafka/test/MockSelector.java
@@ -16,6 +16,14 @@
  */
 package org.apache.kafka.test;
 
+import org.apache.kafka.common.network.ChannelState;
+import org.apache.kafka.common.network.NetworkReceive;
+import org.apache.kafka.common.network.NetworkSend;
+import org.apache.kafka.common.network.Selectable;
+import org.apache.kafka.common.network.Send;
+import org.apache.kafka.common.requests.ByteBufferChannel;
+import org.apache.kafka.common.utils.Time;
+
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
@@ -25,24 +33,17 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.kafka.common.network.ChannelState;
-import org.apache.kafka.common.network.NetworkReceive;
-import org.apache.kafka.common.network.NetworkSend;
-import org.apache.kafka.common.network.Selectable;
-import org.apache.kafka.common.network.Send;
-import org.apache.kafka.common.utils.Time;
-
 /**
  * A fake selector to use for testing
  */
 public class MockSelector implements Selectable {
 
     private final Time time;
-    private final List initiatedSends = new ArrayList();
-    private final List completedSends = new ArrayList();
-    private final List completedReceives = new ArrayList();
+    private final List initiatedSends = new ArrayList<>();
+    private final List completedSends = new ArrayList<>();
+    private final List completedReceives = new ArrayList<>();
     private final Map disconnected = new HashMap<>();
-    private final List connected = new ArrayList();
+    private final List connected = new ArrayList<>();
     private final List delayedReceives = new ArrayList<>();
 
     public MockSelector(Time time) {
@@ -109,8 +110,28 @@ public void send(Send send) {
 
     @Override
     public void poll(long timeout) throws IOException {
-        this.completedSends.addAll(this.initiatedSends);
+        completeInitiatedSends();
+        completeDelayedReceives();
+        time.sleep(timeout);
+    }
+
+    private void completeInitiatedSends() throws IOException {
+        for (Send send : initiatedSends) {
+            completeSend(send);
+        }
         this.initiatedSends.clear();
+    }
+
+    private void completeSend(Send send) throws IOException {
+        // Consume the send so that we will be able to send more requests to the destination
+        ByteBufferChannel discardChannel = new ByteBufferChannel(send.size());
+        while (!send.completed()) {
+            send.writeTo(discardChannel);
+        }
+        completedSends.add(send);
+    }
+
+    private void completeDelayedReceives() {
         for (Send completedSend : completedSends) {
             Iterator delayedReceiveIterator = delayedReceives.iterator();
             while (delayedReceiveIterator.hasNext()) {
@@ -121,7 +142,6 @@ public void poll(long timeout) throws IOException {
                 }
             }
         }
-        time.sleep(timeout);
     }
 
     @Override
@@ -178,4 +198,10 @@ public void unmuteAll() {
     public boolean isChannelReady(String id) {
         return true;
     }
+
+    public void reset() {
+        clear();
+        initiatedSends.clear();
+        delayedReceives.clear();
+    }
 }

From a1b01f48e9ef06291f3e052a37bde887b891e708 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Tue, 6 Mar 2018 00:24:22 -0500
Subject: [PATCH 0115/1847] KAFKA-6309: Improve task assignor load balance
 (#4624)

Sorts TaskIds on first assignment evenly distributing tasks by topicGroupId should help with evening the load of work across topologies. This PR is an initial "strawman" approach which will be followed up (at a later date YTBD) by scoring or assigning weight to processing nodes to ensure even processing distribution.

Added a new test to existing unit test.
---
 .../assignment/StickyTaskAssignor.java        | 10 ++--
 .../assignment/StickyTaskAssignorTest.java    | 55 +++++++++++++++++++
 2 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java
index de8fa57e36a15..5b54d08c03235 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignor.java
@@ -20,10 +20,13 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -106,14 +109,13 @@ private void assignActive() {
         }
 
         // assign any remaining unassigned tasks
-        for (final TaskId taskId : unassigned) {
+        List sortedTasks = new ArrayList<>(unassigned);
+        Collections.sort(sortedTasks);
+        for (final TaskId taskId : sortedTasks) {
             allocateTaskWithClientCandidates(taskId, clients.keySet(), true);
         }
-
     }
 
-
-
     private void allocateTaskWithClientCandidates(final TaskId taskId, final Set clientsWithin, final boolean active) {
         final ClientState client = findClient(taskId, clientsWithin, active);
         taskPairs.addPairs(taskId, client.assignedTasks());
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
index 4f770c862397f..ed22e3c30de84 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/StickyTaskAssignorTest.java
@@ -23,11 +23,13 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.TreeSet;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -350,6 +352,42 @@ public void shouldAssignMoreTasksToClientWithMoreCapacity() {
         assertThat(clients.get(p1).assignedTaskCount(), equalTo(4));
     }
 
+    @Test
+    public void shouldEvenlyDistributeByTaskIdAndPartition() {
+        createClient(p1, 4);
+        createClient(p2, 4);
+        createClient(p3, 4);
+        createClient(p4, 4);
+
+        final List taskIds = new ArrayList<>();
+        final TaskId[] taskIdArray = new TaskId[16];
+
+        for (int i = 1; i <= 2; i++) {
+            for (int j = 0; j < 8; j++) {
+                taskIds.add(new TaskId(i, j));
+            }
+        }
+
+        Collections.shuffle(taskIds);
+        taskIds.toArray(taskIdArray);
+
+        final StickyTaskAssignor taskAssignor = createTaskAssignor(taskIdArray);
+        taskAssignor.assign(0);
+
+        Collections.sort(taskIds);
+        final Set expectedClientOneAssignment = getExpectedTaskIdAssignment(taskIds, 0, 4, 8, 12);
+        final Set expectedClientTwoAssignment = getExpectedTaskIdAssignment(taskIds, 1, 5, 9, 13);
+        final Set expectedClientThreeAssignment = getExpectedTaskIdAssignment(taskIds, 2, 6, 10, 14);
+        final Set expectedClientFourAssignment = getExpectedTaskIdAssignment(taskIds, 3, 7, 11, 15);
+
+        final Map> sortedAssignments = sortClientAssignments(clients);
+
+        assertThat(sortedAssignments.get(p1), equalTo(expectedClientOneAssignment));
+        assertThat(sortedAssignments.get(p2), equalTo(expectedClientTwoAssignment));
+        assertThat(sortedAssignments.get(p3), equalTo(expectedClientThreeAssignment));
+        assertThat(sortedAssignments.get(p4), equalTo(expectedClientFourAssignment));
+    }
+
 
     @Test
     public void shouldNotHaveSameAssignmentOnAnyTwoHosts() {
@@ -665,4 +703,21 @@ private void assertActiveTaskTopicGroupIdsEvenlyDistributed() {
         }
     }
 
+    private Map> sortClientAssignments(final Map clients) {
+        final Map> sortedAssignments = new HashMap<>();
+        for (final Map.Entry entry : clients.entrySet()) {
+            final Set sorted = new TreeSet<>(entry.getValue().activeTasks());
+            sortedAssignments.put(entry.getKey(), sorted);
+        }
+        return sortedAssignments;
+    }
+
+    private Set getExpectedTaskIdAssignment(final List tasks, final int... indices) {
+        final Set sortedAssignment = new TreeSet<>();
+        for (final int index : indices) {
+            sortedAssignment.add(tasks.get(index));
+        }
+        return sortedAssignment;
+    }
+
 }

From d13cbd0cae26d13c36c25d04a489ee309187ebeb Mon Sep 17 00:00:00 2001
From: Ewen Cheslack-Postava 
Date: Mon, 5 Mar 2018 22:21:53 -0800
Subject: [PATCH 0116/1847] KAFKA-3806: Increase offsets retention default to 7
 days (KIP-186) (#4648)

Reviewers: Ismael Juma , Jason Gustafson 
---
 .../main/scala/kafka/server/KafkaConfig.scala |  2 +-
 .../unit/kafka/server/OffsetCommitTest.scala  |  4 +-
 docs/upgrade.html                             | 61 +++++++++++++++++++
 3 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 8b2fb1044e0ea..cf22305caf70c 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -158,7 +158,7 @@ object Defaults {
   val OffsetsTopicPartitions: Int = OffsetConfig.DefaultOffsetsTopicNumPartitions
   val OffsetsTopicSegmentBytes: Int = OffsetConfig.DefaultOffsetsTopicSegmentBytes
   val OffsetsTopicCompressionCodec: Int = OffsetConfig.DefaultOffsetsTopicCompressionCodec.codec
-  val OffsetsRetentionMinutes: Int = 24 * 60
+  val OffsetsRetentionMinutes: Int = 7 * 24 * 60
   val OffsetsRetentionCheckIntervalMs: Long = OffsetConfig.DefaultOffsetsRetentionCheckIntervalMs
   val OffsetCommitTimeoutMs = OffsetConfig.DefaultOffsetCommitTimeoutMs
   val OffsetCommitRequiredAcks = OffsetConfig.DefaultOffsetCommitRequiredAcks
diff --git a/core/src/test/scala/unit/kafka/server/OffsetCommitTest.scala b/core/src/test/scala/unit/kafka/server/OffsetCommitTest.scala
index 0793fa37a600d..3e8a535946814 100755
--- a/core/src/test/scala/unit/kafka/server/OffsetCommitTest.scala
+++ b/core/src/test/scala/unit/kafka/server/OffsetCommitTest.scala
@@ -249,11 +249,11 @@ class OffsetCommitTest extends ZooKeeperTestHarness {
     Thread.sleep(retentionCheckInterval * 2)
     assertEquals(2L, simpleConsumer.fetchOffsets(fetchRequest).requestInfo.get(topicPartition).get.offset)
 
-    // v1 version commit request with commit timestamp set to now - two days
+    // v1 version commit request with commit timestamp set to now - seven + a bit days
     // committed offset should expire
     val commitRequest2 = OffsetCommitRequest(
       groupId = group,
-      requestInfo = immutable.Map(topicPartition -> OffsetAndMetadata(3L, "metadata", Time.SYSTEM.milliseconds - 2*24*60*60*1000L)),
+      requestInfo = immutable.Map(topicPartition -> OffsetAndMetadata(3L, "metadata", Time.SYSTEM.milliseconds - (Defaults.OffsetsRetentionMinutes + 1) * 60 * 1000L)),
       versionId = 1
     )
     assertEquals(Errors.NONE, simpleConsumer.commitOffsets(commitRequest2).commitStatus.get(topicPartition).get)
diff --git a/docs/upgrade.html b/docs/upgrade.html
index 3ac293d8498dd..324f8df1403e6 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -19,6 +19,67 @@
 
 
 
 
 
 
 
 
    
-  
-  
    
-    
-    
-    
    
-  
    
+    
+    
    
+        
+        
+        
    
+    
    
 
    
 
 
 
 
@@ -316,52 +308,52 @@ 
    Hello Kafka Streams
    
 
 
\ No newline at end of file
+
diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
index fdc0af1de1fb1..7ffafb547a872 100644
--- a/docs/streams/upgrade-guide.html
+++ b/docs/streams/upgrade-guide.html
@@ -112,9 +112,8 @@ 
    Streams API
         In addition, in StreamsConfig we have also added default.windowed.key.serde.inner and default.windowed.value.serde.inner
         to let users specify inner serdes if the default serde classes are windowed serdes.
         For more details, see KIP-265.
-    /
    
-
-     
    
+    
    
+    
    
         We have deprecated constructors of KafkaStreams that take a StreamsConfig as parameter.
         Please use the other corresponding constructors that accept java.util.Properties instead.
         For more details, see KIP-245.
@@ -127,6 +126,14 @@ 
    Streams API
       Forwarding based on child index is not supported in the new API any longer.
     
    
 
+    
    
+      Kafka Streams DSL for Scala is a new Kafka Streams client library available for developers authoring Kafka Streams applications in Scala.  It wraps core Kafka Streams DSL types to make it easier to call when
+      interoperating with Scala code.  For example, it includes higher order functions as parameters for transformations avoiding the need anonymous classes in Java 7 or experimental SAM type conversions in Scala 2.11, automatic conversion between Java and Scala collection types, a way
+      to implicitly provide SerDes to reduce boilerplate from your application and make it more typesafe, and more!  For more information see the
+      Kafka Streams DSL for Scala documentation and
+      KIP-270.
+    
    
+
     
    Streams API changes in 1.1.0
    
     
    
         We have added support for methods in ReadOnlyWindowStore which allows for querying WindowStores without the necessity of providing keys.
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index e5f2958ff1086..56abe889e7259 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -60,6 +60,7 @@ versions += [
   log4j: "1.2.17",
   scalaLogging: "3.8.0",
   jaxb: "2.3.0",
+  jfreechart: "1.0.0",
   jopt: "5.0.4",
   junit: "4.12",
   kafka_0100: "0.10.0.1",
@@ -69,6 +70,7 @@ versions += [
   kafka_10: "1.0.1",
   kafka_11: "1.1.0",
   lz4: "1.4.1",
+  mavenArtifact: "3.5.3",
   metrics: "2.2.0",
   // PowerMock 1.x doesn't support Java 9, so use PowerMock 2.0.0 beta
   powermock: "2.0.0-beta.5",
@@ -79,14 +81,11 @@ versions += [
   slf4j: "1.7.25",
   snappy: "1.1.7.1",
   zkclient: "0.10",
-  zookeeper: "3.4.10",
-  jfreechart: "1.0.0",
-  mavenArtifact: "3.5.3"
+  zookeeper: "3.4.10"
 ]
 
 libs += [
   activation: "javax.activation:activation:$versions.activation",
-  argparse4j: "net.sourceforge.argparse4j:argparse4j:$versions.argparse4j",
   apacheda: "org.apache.directory.api:api-all:$versions.apacheda",
   apachedsCoreApi: "org.apache.directory.server:apacheds-core-api:$versions.apacheds",
   apachedsInterceptorKerberos: "org.apache.directory.server:apacheds-interceptor-kerberos:$versions.apacheds",
@@ -96,6 +95,7 @@ libs += [
   apachedsLdifPartition: "org.apache.directory.server:apacheds-ldif-partition:$versions.apacheds",
   apachedsMavibotPartition: "org.apache.directory.server:apacheds-mavibot-partition:$versions.apacheds",
   apachedsJdbmPartition: "org.apache.directory.server:apacheds-jdbm-partition:$versions.apacheds",
+  argparse4j: "net.sourceforge.argparse4j:argparse4j:$versions.argparse4j",
   bcpkix: "org.bouncycastle:bcpkix-jdk15on:$versions.bcpkix",
   easymock: "org.easymock:easymock:$versions.easymock",
   jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:$versions.jackson",
@@ -106,6 +106,7 @@ libs += [
   jettyServlet: "org.eclipse.jetty:jetty-servlet:$versions.jetty",
   jettyServlets: "org.eclipse.jetty:jetty-servlets:$versions.jetty",
   jerseyContainerServlet: "org.glassfish.jersey.containers:jersey-container-servlet:$versions.jersey",
+  jfreechart: "1.0.0",
   jmhCore: "org.openjdk.jmh:jmh-core:$versions.jmh",
   jmhCoreBenchmarks: "org.openjdk.jmh:jmh-core-benchmarks:$versions.jmh",
   jmhGeneratorAnnProcess: "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh",
diff --git a/gradle/findbugs-exclude.xml b/gradle/findbugs-exclude.xml
index 33702056d83f6..f36f32376ab17 100644
--- a/gradle/findbugs-exclude.xml
+++ b/gradle/findbugs-exclude.xml
@@ -305,4 +305,19 @@ For a detailed description of findbugs bug categories, see http://findbugs.sourc
         
         
     
+
+    
+        
+        
+        
+    
+
+-    
+        
+        
+            
+            
+        
+        
+    
 
diff --git a/settings.gradle b/settings.gradle
index 2a7977cfc9343..7082ddd9e05ba 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:test-utils', 'streams:examples',
+include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scala', 'streams:test-utils', 'streams:examples',
         'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102',
         'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11',
         'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file',
diff --git a/streams/streams-scala/.gitignore b/streams/streams-scala/.gitignore
new file mode 100644
index 0000000000000..bf11921177ee2
--- /dev/null
+++ b/streams/streams-scala/.gitignore
@@ -0,0 +1 @@
+/logs/
\ No newline at end of file
diff --git a/streams/streams-scala/NOTICE b/streams/streams-scala/NOTICE
new file mode 100644
index 0000000000000..732e373e1c123
--- /dev/null
+++ b/streams/streams-scala/NOTICE
@@ -0,0 +1,3 @@
+Kafka Streams Scala
+Copyright (C) 2018 Lightbend Inc. 
+Copyright (C) 2017-2018 Alexis Seigneurin.
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/package.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/package.scala
new file mode 100644
index 0000000000000..864fd19f87aaa
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/package.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams
+
+import org.apache.kafka.streams.state.KeyValueStore
+import org.apache.kafka.common.utils.Bytes
+
+package object scala {
+  type ByteArrayKeyValueStore = KeyValueStore[Bytes, Array[Byte]]
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/DefaultSerdes.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/DefaultSerdes.scala
new file mode 100644
index 0000000000000..3f2840eeecbed
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/DefaultSerdes.scala
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import java.nio.ByteBuffer
+
+import org.apache.kafka.common.serialization.{Serde, Serdes}
+import org.apache.kafka.common.utils.Bytes
+import org.apache.kafka.streams.kstream.WindowedSerdes
+
+
+/**
+ * Implicit values for default serdes.
+ * 
    
+ * Bring them in scope for default serializers / de-serializers to work.
+ */
+object DefaultSerdes {
+  implicit val stringSerde: Serde[String] = Serdes.String()
+  implicit val longSerde: Serde[Long] = Serdes.Long().asInstanceOf[Serde[Long]]
+  implicit val byteArraySerde: Serde[Array[Byte]] = Serdes.ByteArray()
+  implicit val bytesSerde: Serde[Bytes] = Serdes.Bytes()
+  implicit val floatSerde: Serde[Float] = Serdes.Float().asInstanceOf[Serde[Float]]
+  implicit val doubleSerde: Serde[Double] = Serdes.Double().asInstanceOf[Serde[Double]]
+  implicit val integerSerde: Serde[Int] = Serdes.Integer().asInstanceOf[Serde[Int]]
+  implicit val shortSerde: Serde[Short] = Serdes.Short().asInstanceOf[Serde[Short]]
+  implicit val byteBufferSerde: Serde[ByteBuffer] = Serdes.ByteBuffer()
+
+  implicit def timeWindowedSerde[T]: WindowedSerdes.TimeWindowedSerde[T] = new WindowedSerdes.TimeWindowedSerde[T]()
+  implicit def sessionWindowedSerde[T]: WindowedSerdes.SessionWindowedSerde[T] = new WindowedSerdes.SessionWindowedSerde[T]()
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala
new file mode 100644
index 0000000000000..9ce9838c7c9a0
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import org.apache.kafka.streams.KeyValue
+import org.apache.kafka.streams.kstream._
+import scala.collection.JavaConverters._
+import java.lang.{Iterable => JIterable}
+
+/**
+ * Implicit classes that offer conversions of Scala function literals to
+ * SAM (Single Abstract Method) objects in Java. These make the Scala APIs much
+ * more expressive, with less boilerplate and more succinct.
+ * 
    
+ * For Scala 2.11, most of these conversions need to be invoked explicitly, as Scala 2.11 does not
+ * have full support for SAM types. 
+ */
+object FunctionConversions {
+
+  implicit class PredicateFromFunction[K, V](val p: (K, V) => Boolean) extends AnyVal {
+    def asPredicate: Predicate[K, V] = new Predicate[K, V] {
+      override def test(key: K, value: V): Boolean = p(key, value)
+    }
+  }
+
+  implicit class MapperFromFunction[T, U, VR](val f:(T,U) => VR) extends AnyVal {
+    def asKeyValueMapper: KeyValueMapper[T, U, VR] = new KeyValueMapper[T, U, VR] {
+      override def apply(key: T, value: U): VR = f(key, value)
+    }
+    def asValueJoiner: ValueJoiner[T, U, VR] = new ValueJoiner[T, U, VR] {
+      override def apply(value1: T, value2: U): VR = f(value1, value2)
+    }
+  }
+
+  implicit class KeyValueMapperFromFunction[K, V, KR, VR](val f:(K,V) => (KR, VR)) extends AnyVal {
+    def asKeyValueMapper: KeyValueMapper[K, V, KeyValue[KR, VR]] = new KeyValueMapper[K, V, KeyValue[KR, VR]] {
+      override def apply(key: K, value: V): KeyValue[KR, VR] = {
+        val (kr, vr) = f(key, value)
+        KeyValue.pair(kr, vr)
+      }
+    }
+  }
+
+  implicit class ValueMapperFromFunction[V, VR](val f: V => VR) extends AnyVal {
+    def asValueMapper: ValueMapper[V, VR] = new ValueMapper[V, VR] {
+      override def apply(value: V): VR = f(value)
+    }
+  }
+
+  implicit class FlatValueMapperFromFunction[V, VR](val f: V => Iterable[VR]) extends AnyVal {
+    def asValueMapper: ValueMapper[V, JIterable[VR]] = new ValueMapper[V, JIterable[VR]] {
+      override def apply(value: V): JIterable[VR] = f(value).asJava
+    }
+  }
+
+  implicit class ValueMapperWithKeyFromFunction[K, V, VR](val f: (K, V) => VR) extends AnyVal {
+    def asValueMapperWithKey: ValueMapperWithKey[K, V, VR] = new ValueMapperWithKey[K, V, VR] {
+      override def apply(readOnlyKey: K, value: V): VR = f(readOnlyKey, value)
+    }
+  }
+
+  implicit class FlatValueMapperWithKeyFromFunction[K, V, VR](val f: (K, V) => Iterable[VR]) extends AnyVal {
+    def asValueMapperWithKey: ValueMapperWithKey[K, V, JIterable[VR]] = new ValueMapperWithKey[K, V, JIterable[VR]] {
+      override def apply(readOnlyKey: K, value: V): JIterable[VR] = f(readOnlyKey, value).asJava
+    }
+  }
+
+  implicit class AggregatorFromFunction[K, V, VA](val f: (K, V, VA) => VA) extends AnyVal {
+    def asAggregator: Aggregator[K, V, VA] = new Aggregator[K, V, VA] {
+      override def apply(key: K, value: V, aggregate: VA): VA = f(key, value, aggregate)
+    }
+  }
+
+  implicit class MergerFromFunction[K,VR](val f: (K, VR, VR) => VR) extends AnyVal {
+    def asMerger: Merger[K, VR] = new Merger[K, VR] {
+      override def apply(aggKey: K, aggOne: VR, aggTwo: VR): VR = f(aggKey, aggOne, aggTwo)
+    }
+  }
+
+  implicit class ReducerFromFunction[V](val f: (V, V) => V) extends AnyVal {
+    def asReducer: Reducer[V] = new Reducer[V] {
+      override def apply(value1: V, value2: V): V = f(value1, value2)
+    }
+  }
+
+  implicit class InitializerFromFunction[VA](val f: () => VA) extends AnyVal {
+    def asInitializer: Initializer[VA] = new Initializer[VA] {
+      override def apply(): VA = f()
+    }
+  }
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala
new file mode 100644
index 0000000000000..000690ba0ab4c
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ImplicitConversions.scala
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import org.apache.kafka.streams.kstream.{KStream => KStreamJ,
+  KTable => KTableJ,
+  KGroupedStream => KGroupedStreamJ,
+  SessionWindowedKStream => SessionWindowedKStreamJ,
+  TimeWindowedKStream => TimeWindowedKStreamJ,
+  KGroupedTable => KGroupedTableJ, _}
+
+import org.apache.kafka.streams.scala.kstream._
+import org.apache.kafka.streams.{KeyValue, Consumed}
+import org.apache.kafka.common.serialization.Serde
+
+import scala.language.implicitConversions
+
+/**
+ * Implicit conversions between the Scala wrapper objects and the underlying Java
+ * objects.
+ */
+object ImplicitConversions {
+
+  implicit def wrapKStream[K, V](inner: KStreamJ[K, V]): KStream[K, V] =
+    new KStream[K, V](inner)
+
+  implicit def wrapKGroupedStream[K, V](inner: KGroupedStreamJ[K, V]): KGroupedStream[K, V] =
+    new KGroupedStream[K, V](inner)
+
+  implicit def wrapSessionWindowedKStream[K, V](inner: SessionWindowedKStreamJ[K, V]): SessionWindowedKStream[K, V] =
+    new SessionWindowedKStream[K, V](inner)
+
+  implicit def wrapTimeWindowedKStream[K, V](inner: TimeWindowedKStreamJ[K, V]): TimeWindowedKStream[K, V] =
+    new TimeWindowedKStream[K, V](inner)
+
+  implicit def wrapKTable[K, V](inner: KTableJ[K, V]): KTable[K, V] =
+    new KTable[K, V](inner)
+
+  implicit def wrapKGroupedTable[K, V](inner: KGroupedTableJ[K, V]): KGroupedTable[K, V] =
+    new KGroupedTable[K, V](inner)
+
+  implicit def tuple2ToKeyValue[K, V](tuple: (K, V)): KeyValue[K, V] = new KeyValue(tuple._1, tuple._2)
+
+  // we would also like to allow users implicit serdes
+  // and these implicits will convert them to `Serialized`, `Produced` or `Consumed`
+
+  implicit def serializedFromSerde[K, V](implicit keySerde: Serde[K], valueSerde: Serde[V]): Serialized[K, V] =
+    Serialized.`with`(keySerde, valueSerde)
+
+  implicit def consumedFromSerde[K, V](implicit keySerde: Serde[K], valueSerde: Serde[V]): Consumed[K, V] =
+    Consumed.`with`(keySerde, valueSerde)
+
+  implicit def producedFromSerde[K, V](implicit keySerde: Serde[K], valueSerde: Serde[V]): Produced[K, V] =
+    Produced.`with`(keySerde, valueSerde)
+
+  implicit def joinedFromKeyValueOtherSerde[K, V, VO]
+    (implicit keySerde: Serde[K], valueSerde: Serde[V], otherValueSerde: Serde[VO]): Joined[K, V, VO] =
+    Joined.`with`(keySerde, valueSerde, otherValueSerde)
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ScalaSerde.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ScalaSerde.scala
new file mode 100644
index 0000000000000..06afcaeed5e06
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/ScalaSerde.scala
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import org.apache.kafka.common.serialization.{Serde, Deserializer => JDeserializer, Serializer => JSerializer}
+
+trait ScalaSerde[T] extends Serde[T] {
+  override def deserializer(): JDeserializer[T]
+
+  override def serializer(): JSerializer[T]
+
+  override def configure(configs: java.util.Map[String, _], isKey: Boolean): Unit = ()
+
+  override def close(): Unit = ()
+}
+
+trait SimpleScalaSerde[T >: Null] extends Serde[T] with ScalaSerde[T] {
+  def serialize(data: T): Array[Byte]
+  def deserialize(data: Array[Byte]): Option[T]
+
+  private def outerSerialize(data: T): Array[Byte] = serialize(data)
+  private def outerDeserialize(data: Array[Byte]): Option[T] = deserialize(data)
+
+  override def deserializer(): Deserializer[T] = new Deserializer[T] {
+    override def deserialize(data: Array[Byte]): Option[T] = outerDeserialize(data)
+  }
+
+  override def serializer(): Serializer[T] = new Serializer[T] {
+    override def serialize(data: T): Array[Byte] = outerSerialize(data)
+  }
+}
+
+trait Deserializer[T >: Null] extends JDeserializer[T] {
+  override def configure(configs: java.util.Map[String, _], isKey: Boolean): Unit = ()
+
+  override def close(): Unit = ()
+
+  override def deserialize(topic: String, data: Array[Byte]): T =
+    Option(data).flatMap(deserialize).orNull
+
+  def deserialize(data: Array[Byte]): Option[T]
+}
+
+trait Serializer[T] extends JSerializer[T] {
+  override def configure(configs: java.util.Map[String, _], isKey: Boolean): Unit = ()
+
+  override def close(): Unit = ()
+
+  override def serialize(topic: String, data: T): Array[Byte] =
+    Option(data).map(serialize).orNull
+
+  def serialize(data: T): Array[Byte]
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala
new file mode 100644
index 0000000000000..9e6e204a37292
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/StreamsBuilder.scala
@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import java.util.regex.Pattern
+
+import org.apache.kafka.streams.kstream.{GlobalKTable, Materialized}
+import org.apache.kafka.streams.processor.{ProcessorSupplier, StateStore}
+import org.apache.kafka.streams.state.StoreBuilder
+import org.apache.kafka.streams.{Consumed, StreamsBuilder => StreamsBuilderJ, Topology}
+
+import org.apache.kafka.streams.scala.kstream._
+import ImplicitConversions._
+import scala.collection.JavaConverters._
+
+/**
+  * Wraps the Java class StreamsBuilder and delegates method calls to the underlying Java object.
+  */
+class StreamsBuilder(inner: StreamsBuilderJ = new StreamsBuilderJ) {
+
+  /**
+   * Create a [[kstream.KStream]] from the specified topic.
+   * 
    
+   * The `implicit Consumed` instance provides the values of `auto.offset.reset` strategy, `TimestampExtractor`, 
+   * key and value deserializers etc. If the implicit is not found in scope, compiler error will result.
+   * 
    
+   * A convenient alternative is to have the necessary implicit serdes in scope, which will be implicitly
+   * converted to generate an instance of `Consumed`. @see [[ImplicitConversions]]. 
+   * {{{
+   * // Brings all implicit conversions in scope
+   * import ImplicitConversions._
+   *
+   * // Bring implicit default serdes in scope
+   * import DefaultSerdes._
+   *
+   * val builder = new StreamsBuilder()
+   *
+   * // stream function gets the implicit Consumed which is constructed automatically
+   * // from the serdes through the implicits in ImplicitConversions#consumedFromSerde
+   * val userClicksStream: KStream[String, Long] = builder.stream(userClicksTopic)
+   * }}}
+   *
+   * @param topic the topic name
+   * @return a [[kstream.KStream]] for the specified topic
+   */
+  def stream[K, V](topic: String)(implicit consumed: Consumed[K, V]): KStream[K, V] =
+    inner.stream[K, V](topic, consumed)
+
+  /**
+   * Create a [[kstream.KStream]] from the specified topics.
+   *
+   * @param topics the topic names
+   * @return a [[kstream.KStream]] for the specified topics
+   * @see #stream(String)
+   * @see `org.apache.kafka.streams.StreamsBuilder#stream`
+   */
+  def stream[K, V](topics: List[String])(implicit consumed: Consumed[K, V]): KStream[K, V] =
+    inner.stream[K, V](topics.asJava, consumed)
+
+  /**
+   * Create a [[kstream.KStream]] from the specified topic pattern.
+   *
+   * @param topics the topic name pattern
+   * @return a [[kstream.KStream]] for the specified topics
+   * @see #stream(String)
+   * @see `org.apache.kafka.streams.StreamsBuilder#stream`
+   */
+  def stream[K, V](topicPattern: Pattern)(implicit consumed: Consumed[K, V]): KStream[K, V] =
+    inner.stream[K, V](topicPattern, consumed)
+
+  /**
+   * Create a [[kstream.KTable]] from the specified topic.
+   * 
    
+   * The `implicit Consumed` instance provides the values of `auto.offset.reset` strategy, `TimestampExtractor`, 
+   * key and value deserializers etc. If the implicit is not found in scope, compiler error will result.
+   * 
    
+   * A convenient alternative is to have the necessary implicit serdes in scope, which will be implicitly
+   * converted to generate an instance of `Consumed`. @see [[ImplicitConversions]]. 
+   * {{{
+   * // Brings all implicit conversions in scope
+   * import ImplicitConversions._
+   *
+   * // Bring implicit default serdes in scope
+   * import DefaultSerdes._
+   *
+   * val builder = new StreamsBuilder()
+   *
+   * // stream function gets the implicit Consumed which is constructed automatically
+   * // from the serdes through the implicits in ImplicitConversions#consumedFromSerde
+   * val userClicksStream: KTable[String, Long] = builder.table(userClicksTopic)
+   * }}}
+   *
+   * @param topic the topic name
+   * @return a [[kstream.KTable]] for the specified topic
+   * @see `org.apache.kafka.streams.StreamsBuilder#table`
+   */
+  def table[K, V](topic: String)(implicit consumed: Consumed[K, V]): KTable[K, V] =
+    inner.table[K, V](topic, consumed)
+
+  /**
+   * Create a [[kstream.KTable]] from the specified topic.
+   *
+   * @param topic the topic name
+   * @param materialized  the instance of `Materialized` used to materialize a state store
+   * @return a [[kstream.KTable]] for the specified topic
+   * @see #table(String)
+   * @see `org.apache.kafka.streams.StreamsBuilder#table`
+   */
+  def table[K, V](topic: String, materialized: Materialized[K, V, ByteArrayKeyValueStore])
+    (implicit consumed: Consumed[K, V]): KTable[K, V] =
+    inner.table[K, V](topic, consumed, materialized)
+
+  /**
+   * Create a `GlobalKTable` from the specified topic. The serializers from the implicit `Consumed`
+   * instance will be used. Input records with `null` key will be dropped.
+   *
+   * @param topic the topic name
+   * @return a `GlobalKTable` for the specified topic
+   * @see `org.apache.kafka.streams.StreamsBuilder#globalTable`
+   */
+  def globalTable[K, V](topic: String)(implicit consumed: Consumed[K, V]): GlobalKTable[K, V] =
+    inner.globalTable(topic, consumed)
+
+  /**
+   * Create a `GlobalKTable` from the specified topic. The resulting `GlobalKTable` will be materialized 
+   * in a local `KeyValueStore` configured with the provided instance of `Materialized`. The serializers 
+   * from the implicit `Consumed` instance will be used.
+   *
+   * @param topic the topic name
+   * @param materialized  the instance of `Materialized` used to materialize a state store
+   * @return a `GlobalKTable` for the specified topic
+   * @see `org.apache.kafka.streams.StreamsBuilder#globalTable`
+   */
+  def globalTable[K, V](topic: String, materialized: Materialized[K, V, ByteArrayKeyValueStore])
+    (implicit consumed: Consumed[K, V]): GlobalKTable[K, V] =
+    inner.globalTable(topic, consumed, materialized)
+
+  /**
+   * Adds a state store to the underlying `Topology`. The store must still be "connected" to a `Processor`, 
+   * `Transformer`, or `ValueTransformer` before it can be used.
+   *
+   * @param builder the builder used to obtain this state store `StateStore` instance
+   * @return the underlying Java abstraction `StreamsBuilder` after adding the `StateStore`
+   * @throws TopologyException if state store supplier is already added
+   * @see `org.apache.kafka.streams.StreamsBuilder#addStateStore`
+   */
+  def addStateStore(builder: StoreBuilder[_ <: StateStore]): StreamsBuilderJ = inner.addStateStore(builder)
+
+  /**
+   * Adds a global `StateStore` to the topology. Global stores should not be added to `Processor, `Transformer`, 
+   * or `ValueTransformer` (in contrast to regular stores).
+   *
+   * @see `org.apache.kafka.streams.StreamsBuilder#addGlobalStore`
+   */ 
+  def addGlobalStore(storeBuilder: StoreBuilder[_ <: StateStore],
+                     topic: String,
+                     consumed: Consumed[_, _],
+                     stateUpdateSupplier: ProcessorSupplier[_, _]): StreamsBuilderJ =
+    inner.addGlobalStore(storeBuilder, topic, consumed, stateUpdateSupplier)
+
+  def build(): Topology = inner.build()
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KGroupedStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KGroupedStream.scala
new file mode 100644
index 0000000000000..8f0ae93c1491d
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KGroupedStream.scala
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+package kstream
+
+import org.apache.kafka.streams.kstream.{KGroupedStream => KGroupedStreamJ, _}
+import org.apache.kafka.common.serialization.Serde
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.FunctionConversions._
+
+
+/**
+ * Wraps the Java class KGroupedStream and delegates method calls to the underlying Java object.
+ *
+ * @param [K] Type of keys
+ * @param [V] Type of values
+ * @param inner The underlying Java abstraction for KGroupedStream
+ *
+ * @see `org.apache.kafka.streams.kstream.KGroupedStream`
+ */
+class KGroupedStream[K, V](val inner: KGroupedStreamJ[K, V]) {
+
+  /**
+   * Count the number of records in this stream by the grouped key.
+   *
+   * @return a [[KTable]] that contains "update" records with unmodified keys and `Long` values that
+   * represent the latest (rolling) count (i.e., number of records) for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#count`
+   */ 
+  def count(): KTable[K, Long] = {
+    val c: KTable[K, java.lang.Long] = inner.count()
+    c.mapValues[Long](Long2long _)
+  }
+
+  /**
+   * Count the number of records in this stream by the grouped key.
+   * The result is written into a local `KeyValueStore` (which is basically an ever-updating materialized view)
+   * provided by the given `materialized`.
+   *
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys and `Long` values that
+   * represent the latest (rolling) count (i.e., number of records) for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#count`
+   */ 
+  def count(materialized: Materialized[K, Long, ByteArrayKeyValueStore]): KTable[K, Long] = { 
+    val c: KTable[K, java.lang.Long] = inner.count(materialized.asInstanceOf[Materialized[K, java.lang.Long, ByteArrayKeyValueStore]])
+    c.mapValues[Long](Long2long _)
+  }
+
+  /**
+   * Combine the values of records in this stream by the grouped key.
+   *
+   * @param reducer   a function `(V, V) => V` that computes a new aggregate result. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#reduce`
+   */ 
+  def reduce(reducer: (V, V) => V): KTable[K, V] = {
+    inner.reduce(reducer.asReducer)
+  }
+
+  /**
+   * Combine the values of records in this stream by the grouped key.
+   *
+   * @param reducer   a function `(V, V) => V` that computes a new aggregate result. 
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#reduce`
+   */ 
+  def reduce(reducer: (V, V) => V,
+    materialized: Materialized[K, V, ByteArrayKeyValueStore]): KTable[K, V] = {
+
+    // need this explicit asReducer for Scala 2.11 or else the SAM conversion doesn't take place
+    // works perfectly with Scala 2.12 though
+    inner.reduce(((v1: V, v2: V) => reducer(v1, v2)).asReducer, materialized)
+  }
+
+  /**
+   * Aggregate the values of records in this stream by the grouped key.
+   *
+   * @param initializer   an `Initializer` that computes an initial intermediate aggregation result
+   * @param aggregator    an `Aggregator` that computes a new aggregate result
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#aggregate`
+   */ 
+  def aggregate[VR](initializer: () => VR,
+    aggregator: (K, V, VR) => VR): KTable[K, VR] = {
+    inner.aggregate(initializer.asInitializer, aggregator.asAggregator)
+  }
+
+  /**
+   * Aggregate the values of records in this stream by the grouped key.
+   *
+   * @param initializer   an `Initializer` that computes an initial intermediate aggregation result
+   * @param aggregator    an `Aggregator` that computes a new aggregate result
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#aggregate`
+   */ 
+  def aggregate[VR](initializer: () => VR,
+    aggregator: (K, V, VR) => VR,
+    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+    inner.aggregate(initializer.asInitializer, aggregator.asAggregator, materialized)
+  }
+
+  /**
+   * Create a new [[SessionWindowedKStream]] instance that can be used to perform session windowed aggregations.
+   *
+   * @param windows the specification of the aggregation `SessionWindows`
+   * @return an instance of [[SessionWindowedKStream]]
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#windowedBy`
+   */
+  def windowedBy(windows: SessionWindows): SessionWindowedKStream[K, V] =
+    inner.windowedBy(windows)
+
+  /**
+   * Create a new [[TimeWindowedKStream]] instance that can be used to perform windowed aggregations.
+   *
+   * @param windows the specification of the aggregation `Windows`
+   * @return an instance of [[TimeWindowedKStream]]
+   * @see `org.apache.kafka.streams.kstream.KGroupedStream#windowedBy`
+   */
+  def windowedBy[W <: Window](windows: Windows[W]): TimeWindowedKStream[K, V] =
+    inner.windowedBy(windows)
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KGroupedTable.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KGroupedTable.scala
new file mode 100644
index 0000000000000..57c44fc8033e6
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KGroupedTable.scala
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+package kstream
+
+import org.apache.kafka.streams.kstream.{KGroupedTable => KGroupedTableJ, _}
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.FunctionConversions._
+
+/**
+ * Wraps the Java class KGroupedTable and delegates method calls to the underlying Java object.
+ *
+ * @param [K] Type of keys
+ * @param [V] Type of values
+ * @param inner The underlying Java abstraction for KGroupedTable
+ *
+ * @see `org.apache.kafka.streams.kstream.KGroupedTable`
+ */
+class KGroupedTable[K, V](inner: KGroupedTableJ[K, V]) {
+
+  /**
+   * Count number of records of the original [[KTable]] that got [[KTable#groupBy]] to
+   * the same key into a new instance of [[KTable]].
+   *
+   * @return a [[KTable]] that contains "update" records with unmodified keys and `Long` values that
+   * represent the latest (rolling) count (i.e., number of records) for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedTable#count`
+   */ 
+  def count(): KTable[K, Long] = {
+    val c: KTable[K, java.lang.Long] = inner.count()
+    c.mapValues[Long](Long2long(_))
+  }
+
+  /**
+   * Count number of records of the original [[KTable]] that got [[KTable#groupBy]] to
+   * the same key into a new instance of [[KTable]].
+   *
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys and `Long` values that
+   * represent the latest (rolling) count (i.e., number of records) for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedTable#count`
+   */ 
+  def count(materialized: Materialized[K, Long, ByteArrayKeyValueStore]): KTable[K, Long] =
+    inner.count(materialized)
+
+  /**
+   * Combine the value of records of the original [[KTable]] that got [[KTable#groupBy]]
+   * to the same key into a new instance of [[KTable]].
+   *
+   * @param adder      a function that adds a new value to the aggregate result
+   * @param subtractor a function that removed an old value from the aggregate result
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedTable#reduce`
+   */
+  def reduce(adder: (V, V) => V,
+             subtractor: (V, V) => V): KTable[K, V] = {
+
+    // need this explicit asReducer for Scala 2.11 or else the SAM conversion doesn't take place
+    // works perfectly with Scala 2.12 though
+    inner.reduce(adder.asReducer, subtractor.asReducer)
+  }
+
+  /**
+   * Combine the value of records of the original [[KTable]] that got [[KTable#groupBy]]
+   * to the same key into a new instance of [[KTable]].
+   *
+   * @param adder      a function that adds a new value to the aggregate result
+   * @param subtractor a function that removed an old value from the aggregate result
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedTable#reduce`
+   */
+  def reduce(adder: (V, V) => V,
+             subtractor: (V, V) => V,
+             materialized: Materialized[K, V, ByteArrayKeyValueStore]): KTable[K, V] = {
+
+    // need this explicit asReducer for Scala 2.11 or else the SAM conversion doesn't take place
+    // works perfectly with Scala 2.12 though
+    inner.reduce(adder.asReducer, subtractor.asReducer, materialized)
+  }
+
+  /**
+   * Aggregate the value of records of the original [[KTable]] that got [[KTable#groupBy]]
+   * to the same key into a new instance of [[KTable]] using default serializers and deserializers.
+   *
+   * @param initializer a function that provides an initial aggregate result value
+   * @param adder       a function that adds a new record to the aggregate result
+   * @param subtractor  an aggregator function that removed an old record from the aggregate result
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedTable#aggregate`
+   */
+  def aggregate[VR](initializer: () => VR,
+                    adder: (K, V, VR) => VR,
+                    subtractor: (K, V, VR) => VR): KTable[K, VR] = {
+
+    inner.aggregate(initializer.asInitializer, adder.asAggregator, subtractor.asAggregator)
+  }
+
+  /**
+   * Aggregate the value of records of the original [[KTable]] that got [[KTable#groupBy]]
+   * to the same key into a new instance of [[KTable]] using default serializers and deserializers.
+   *
+   * @param initializer a function that provides an initial aggregate result value
+   * @param adder       a function that adds a new record to the aggregate result
+   * @param subtractor  an aggregator function that removed an old record from the aggregate result
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.KGroupedTable#aggregate`
+   */
+  def aggregate[VR](initializer: () => VR,
+                    adder: (K, V, VR) => VR,
+                    subtractor: (K, V, VR) => VR,
+                    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+
+    inner.aggregate(initializer.asInitializer, adder.asAggregator, subtractor.asAggregator, materialized)
+  }
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
new file mode 100644
index 0000000000000..94c36add172e4
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
@@ -0,0 +1,581 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+package kstream
+
+import org.apache.kafka.streams.KeyValue
+import org.apache.kafka.streams.kstream.{KStream => KStreamJ, _}
+import org.apache.kafka.streams.processor.{Processor, ProcessorContext, ProcessorSupplier}
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.FunctionConversions._
+
+import scala.collection.JavaConverters._
+
+/**
+ * Wraps the Java class [[org.apache.kafka.streams.kstream.KStream]] and delegates method calls to the underlying Java object.
+ *
+ * @param [K] Type of keys
+ * @param [V] Type of values
+ * @param inner The underlying Java abstraction for KStream
+ *
+ * @see `org.apache.kafka.streams.kstream.KStream`
+ */
+class KStream[K, V](val inner: KStreamJ[K, V]) {
+
+  /**
+   * Create a new [[KStream]] that consists all records of this stream which satisfies the given
+   * predicate
+   *
+   * @param predicate a filter that is applied to each record
+   * @return a [[KStream]] that contains only those records that satisfy the given predicate
+   * @see `org.apache.kafka.streams.kstream.KStream#filter`
+   */ 
+  def filter(predicate: (K, V) => Boolean): KStream[K, V] = {
+    inner.filter(predicate.asPredicate)
+  }
+
+  /**
+   * Create a new [[KStream]] that consists all records of this stream which do not satisfy the given
+   * predicate
+   *
+   * @param predicate a filter that is applied to each record
+   * @return a [[KStream]] that contains only those records that do not satisfy the given predicate
+   * @see `org.apache.kafka.streams.kstream.KStream#filterNot`
+   */ 
+  def filterNot(predicate: (K, V) => Boolean): KStream[K, V] = {
+    inner.filterNot(predicate.asPredicate)
+  }
+
+  /**
+   * Set a new key (with possibly new type) for each input record.
+   * 
    
+   * The function `mapper` passed is applied to every record and results in the generation of a new
+   * key `KR`. The function outputs a new [[KStream]] where each record has this new key.
+   *
+   * @param mapper a function `(K, V) => KR` that computes a new key for each record
+   * @return a [[KStream]] that contains records with new key (possibly of different type) and unmodified value
+   * @see `org.apache.kafka.streams.kstream.KStream#selectKey`
+   */ 
+  def selectKey[KR](mapper: (K, V) => KR): KStream[KR, V] = {
+    inner.selectKey[KR](mapper.asKeyValueMapper)
+  }
+
+  /**
+   * Transform each record of the input stream into a new record in the output stream (both key and value type can be
+   * altered arbitrarily).
+   * 
    
+   * The provided `mapper`, a function `(K, V) => (KR, VR)` is applied to each input record and computes a new output record.
+   *
+   * @param mapper a function `(K, V) => (KR, VR)` that computes a new output record
+   * @return a [[KStream]] that contains records with new key and value (possibly both of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#map`
+   */ 
+  def map[KR, VR](mapper: (K, V) => (KR, VR)): KStream[KR, VR] = {
+    val kvMapper = mapper.tupled andThen tuple2ToKeyValue
+    inner.map[KR, VR](((k: K, v: V) => kvMapper(k, v)).asKeyValueMapper)
+  }
+
+  /**
+   * Transform the value of each input record into a new value (with possible new type) of the output record.
+   * 
    
+   * The provided `mapper`, a function `V => VR` is applied to each input record value and computes a new value for it
+   *
+   * @param mapper, a function `V => VR` that computes a new output value
+   * @return a [[KStream]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#mapValues`
+   */ 
+  def mapValues[VR](mapper: V => VR): KStream[K, VR] = {
+    inner.mapValues[VR](mapper.asValueMapper)
+  }
+
+  /**
+   * Transform the value of each input record into a new value (with possible new type) of the output record.
+   * 
    
+   * The provided `mapper`, a function `(K, V) => VR` is applied to each input record value and computes a new value for it
+   *
+   * @param mapper, a function `(K, V) => VR` that computes a new output value
+   * @return a [[KStream]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#mapValues`
+   */ 
+  def mapValues[VR](mapper: (K, V) => VR): KStream[K, VR] = {
+    inner.mapValues[VR](mapper.asValueMapperWithKey)
+  }
+
+  /**
+   * Transform each record of the input stream into zero or more records in the output stream (both key and value type
+   * can be altered arbitrarily).
+   * 
    
+   * The provided `mapper`, function `(K, V) => Iterable[(KR, VR)]` is applied to each input record and computes zero or more output records.
+   *
+   * @param mapper function `(K, V) => Iterable[(KR, VR)]` that computes the new output records
+   * @return a [[KStream]] that contains more or less records with new key and value (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#flatMap`
+   */ 
+  def flatMap[KR, VR](mapper: (K, V) => Iterable[(KR, VR)]): KStream[KR, VR] = {
+    val kvMapper = mapper.tupled andThen (iter => iter.map(tuple2ToKeyValue).asJava)
+    inner.flatMap[KR, VR](((k: K, v: V) => kvMapper(k , v)).asKeyValueMapper)
+  }
+
+  /**
+   * Create a new [[KStream]] by transforming the value of each record in this stream into zero or more values
+   * with the same key in the new stream.
+   * 
    
+   * Transform the value of each input record into zero or more records with the same (unmodified) key in the output
+   * stream (value type can be altered arbitrarily).
+   * The provided `mapper`, a function `V => Iterable[VR]` is applied to each input record and computes zero or more output values.
+   *
+   * @param mapper a function `V => Iterable[VR]` that computes the new output values
+   * @return a [[KStream]] that contains more or less records with unmodified keys and new values of different type
+   * @see `org.apache.kafka.streams.kstream.KStream#flatMapValues`
+   */ 
+  def flatMapValues[VR](mapper: V => Iterable[VR]): KStream[K, VR] = {
+    inner.flatMapValues[VR](mapper.asValueMapper)
+  }
+
+  /**
+   * Create a new [[KStream]] by transforming the value of each record in this stream into zero or more values
+   * with the same key in the new stream.
+   * 
    
+   * Transform the value of each input record into zero or more records with the same (unmodified) key in the output
+   * stream (value type can be altered arbitrarily).
+   * The provided `mapper`, a function `(K, V) => Iterable[VR]` is applied to each input record and computes zero or more output values.
+   *
+   * @param mapper a function `(K, V) => Iterable[VR]` that computes the new output values
+   * @return a [[KStream]] that contains more or less records with unmodified keys and new values of different type
+   * @see `org.apache.kafka.streams.kstream.KStream#flatMapValues`
+   */ 
+  def flatMapValues[VR](mapper: (K, V) => Iterable[VR]): KStream[K, VR] = {
+    inner.flatMapValues[VR](mapper.asValueMapperWithKey)
+  }
+
+  /**
+   * Print the records of this KStream using the options provided by `Printed`
+   *
+   * @param printed options for printing
+   * @see `org.apache.kafka.streams.kstream.KStream#print`
+   */
+  def print(printed: Printed[K, V]): Unit = inner.print(printed)
+
+  /**
+   * Perform an action on each record of 'KStream`
+   *
+   * @param action an action to perform on each record
+   * @see `org.apache.kafka.streams.kstream.KStream#foreach`
+   */
+  def foreach(action: (K, V) => Unit): Unit = {
+    inner.foreach((k: K, v: V) => action(k, v))
+  }
+
+  /**
+   * Creates an array of {@code KStream} from this stream by branching the records in the original stream based on
+   * the supplied predicates.
+   *
+   * @param predicates the ordered list of functions that return a Boolean
+   * @return multiple distinct substreams of this [[KStream]]
+   * @see `org.apache.kafka.streams.kstream.KStream#branch`
+   */
+  def branch(predicates: ((K, V) => Boolean)*): Array[KStream[K, V]] = {
+    inner.branch(predicates.map(_.asPredicate): _*).map(kstream => wrapKStream(kstream))
+  }
+
+  /**
+   * Materialize this stream to a topic and creates a new [[KStream]] from the topic using the `Produced` instance for 
+   * configuration of the `Serde key serde`, `Serde value serde`, and `StreamPartitioner`
+   * 
    
+   * The user can either supply the `Produced` instance as an implicit in scope or she can also provide implicit
+   * key and value serdes that will be converted to a `Produced` instance implicitly.
+   * 
    
+   * {{{
+   * Example:
+   *
+   * // brings implicit serdes in scope
+   * import DefaultSerdes._
+   *
+   * //..
+   * val clicksPerRegion: KTable[String, Long] = //..
+   *
+   * // Implicit serdes in scope will generate an implicit Produced instance, which
+   * // will be passed automatically to the call of through below
+   * clicksPerRegion.through(topic)
+   *
+   * // Similarly you can create an implicit Produced and it will be passed implicitly
+   * // to the through call
+   * }}}
+   *
+   * @param topic the topic name
+   * @param (implicit) produced the instance of Produced that gives the serdes and `StreamPartitioner`
+   * @return a [[KStream]] that contains the exact same (and potentially repartitioned) records as this [[KStream]]
+   * @see `org.apache.kafka.streams.kstream.KStream#through`
+   */
+  def through(topic: String)(implicit produced: Produced[K, V]): KStream[K, V] =
+    inner.through(topic, produced)
+
+  /**
+   * Materialize this stream to a topic using the `Produced` instance for 
+   * configuration of the `Serde key serde`, `Serde value serde`, and `StreamPartitioner`
+   * 
    
+   * The user can either supply the `Produced` instance as an implicit in scope or she can also provide implicit
+   * key and value serdes that will be converted to a `Produced` instance implicitly.
+   * 
    
+   * {{{
+   * Example:
+   *
+   * // brings implicit serdes in scope
+   * import DefaultSerdes._
+   *
+   * //..
+   * val clicksPerRegion: KTable[String, Long] = //..
+   *
+   * // Implicit serdes in scope will generate an implicit Produced instance, which
+   * // will be passed automatically to the call of through below
+   * clicksPerRegion.to(topic)
+   *
+   * // Similarly you can create an implicit Produced and it will be passed implicitly
+   * // to the through call
+   * }}}
+   *
+   * @param topic the topic name
+   * @param (implicit) produced the instance of Produced that gives the serdes and `StreamPartitioner`
+   * @see `org.apache.kafka.streams.kstream.KStream#to`
+   */
+  def to(topic: String)(implicit produced: Produced[K, V]): Unit =
+    inner.to(topic, produced)
+
+  /**
+   * Transform each record of the input stream into zero or more records in the output stream (both key and value type
+   * can be altered arbitrarily).
+   * A `Transformer` (provided by the given `TransformerSupplier`) is applied to each input record and
+   * computes zero or more output records. In order to assign a state, the state must be created and registered 
+   * beforehand via stores added via `addStateStore` or `addGlobalStore` before they can be connected to the `Transformer` 
+   *
+   * @param transformerSupplier a instance of `TransformerSupplier` that generates a `Transformer`
+   * @param stateStoreNames     the names of the state stores used by the processor
+   * @return a [[KStream]] that contains more or less records with new key and value (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#transform`
+   */ 
+  def transform[K1, V1](transformerSupplier: Transformer[K, V, (K1, V1)],
+    stateStoreNames: String*): KStream[K1, V1] = {
+    val transformerSupplierJ: TransformerSupplier[K, V, KeyValue[K1, V1]] = new TransformerSupplier[K, V, KeyValue[K1, V1]] {
+      override def get(): Transformer[K, V, KeyValue[K1, V1]] = {
+        new Transformer[K, V, KeyValue[K1, V1]] {
+          override def transform(key: K, value: V): KeyValue[K1, V1] = {
+            transformerSupplier.transform(key, value) match {
+              case (k1, v1) => KeyValue.pair(k1, v1)
+              case _ => null
+            }
+          }
+
+          override def init(context: ProcessorContext): Unit = transformerSupplier.init(context)
+
+          @deprecated ("Please use Punctuator functional interface at https://kafka.apache.org/10/javadoc/org/apache/kafka/streams/processor/Punctuator.html instead", "0.1.3") // scalastyle:ignore
+          override def punctuate(timestamp: Long): KeyValue[K1, V1] = {
+            transformerSupplier.punctuate(timestamp) match {
+              case (k1, v1) => KeyValue.pair[K1, V1](k1, v1)
+              case _ => null
+            }
+          }
+
+          override def close(): Unit = transformerSupplier.close()
+        }
+      }
+    }
+    inner.transform(transformerSupplierJ, stateStoreNames: _*)
+  }
+
+  /**
+   * Transform the value of each input record into a new value (with possible new type) of the output record.
+   * A `ValueTransformer` (provided by the given `ValueTransformerSupplier`) is applied to each input
+   * record value and computes a new value for it.
+   * In order to assign a state, the state must be created and registered 
+   * beforehand via stores added via `addStateStore` or `addGlobalStore` before they can be connected to the `Transformer` 
+   *
+   * @param valueTransformerSupplier a instance of `ValueTransformerSupplier` that generates a `ValueTransformer`
+   * @param stateStoreNames          the names of the state stores used by the processor
+   * @return a [[KStream]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#transformValues`
+   */ 
+  def transformValues[VR](valueTransformerSupplier: ValueTransformerSupplier[V, VR],
+                          stateStoreNames: String*): KStream[K, VR] = {
+    inner.transformValues[VR](valueTransformerSupplier, stateStoreNames: _*)
+  }
+
+  /**
+   * Transform the value of each input record into a new value (with possible new type) of the output record.
+   * A `ValueTransformer` (provided by the given `ValueTransformerSupplier`) is applied to each input
+   * record value and computes a new value for it.
+   * In order to assign a state, the state must be created and registered 
+   * beforehand via stores added via `addStateStore` or `addGlobalStore` before they can be connected to the `Transformer` 
+   *
+   * @param valueTransformerSupplier a instance of `ValueTransformerWithKeySupplier` that generates a `ValueTransformerWithKey`
+   * @param stateStoreNames          the names of the state stores used by the processor
+   * @return a [[KStream]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KStream#transformValues`
+   */ 
+  def transformValues[VR](valueTransformerWithKeySupplier: ValueTransformerWithKeySupplier[K, V, VR],
+                          stateStoreNames: String*): KStream[K, VR] = {
+    inner.transformValues[VR](valueTransformerWithKeySupplier, stateStoreNames: _*)
+  }
+
+  /**
+   * Process all records in this stream, one record at a time, by applying a `Processor` (provided by the given
+   * `processorSupplier`).
+   * In order to assign a state, the state must be created and registered 
+   * beforehand via stores added via `addStateStore` or `addGlobalStore` before they can be connected to the `Transformer` 
+   *
+   * @param processorSupplier a function that generates a [[org.apache.kafka.stream.Processor]]
+   * @param stateStoreNames   the names of the state store used by the processor
+   * @see `org.apache.kafka.streams.kstream.KStream#process`
+   */ 
+  def process(processorSupplier: () => Processor[K, V],
+    stateStoreNames: String*): Unit = {
+
+    val processorSupplierJ: ProcessorSupplier[K, V] = new ProcessorSupplier[K, V] {
+      override def get(): Processor[K, V] = processorSupplier()
+    }
+    inner.process(processorSupplierJ, stateStoreNames: _*)
+  }
+
+  /**
+   * Group the records by their current key into a [[KGroupedStream]] 
+   * 
    
+   * The user can either supply the `Serialized` instance as an implicit in scope or she can also provide an implicit
+   * serdes that will be converted to a `Serialized` instance implicitly.
+   * 
    
+   * {{{
+   * Example:
+   *
+   * // brings implicit serdes in scope
+   * import DefaultSerdes._
+   *
+   * val clicksPerRegion: KTable[String, Long] =
+   *   userClicksStream
+   *     .leftJoin(userRegionsTable, (clicks: Long, region: String) => (if (region == null) "UNKNOWN" else region, clicks))
+   *     .map((_, regionWithClicks) => regionWithClicks)
+   *
+   *     // the groupByKey gets the Serialized instance through an implicit conversion of the
+   *     // serdes brought into scope through the import DefaultSerdes._ above
+   *     .groupByKey
+   *     .reduce(_ + _)
+   *
+   * // Similarly you can create an implicit Serialized and it will be passed implicitly
+   * // to the groupByKey call
+   * }}}
+   *
+   * @param (implicit) serialized the instance of Serialized that gives the serdes 
+   * @return a [[KGroupedStream]] that contains the grouped records of the original [[KStream]]
+   * @see `org.apache.kafka.streams.kstream.KStream#groupByKey`
+   */ 
+  def groupByKey(implicit serialized: Serialized[K, V]): KGroupedStream[K, V] =
+    inner.groupByKey(serialized)
+
+  /**
+   * Group the records of this [[KStream]] on a new key that is selected using the provided key transformation function
+   * and the `Serialized` instance.
+   * 
    
+   * The user can either supply the `Serialized` instance as an implicit in scope or she can also provide an implicit
+   * serdes that will be converted to a `Serialized` instance implicitly.
+   * 
    
+   * {{{
+   * Example:
+   *
+   * // brings implicit serdes in scope
+   * import DefaultSerdes._
+   *
+   * val textLines = streamBuilder.stream[String, String](inputTopic)
+   *
+   * val pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS)
+   *
+   * val wordCounts: KTable[String, Long] =
+   *   textLines.flatMapValues(v => pattern.split(v.toLowerCase))
+   *
+   *     // the groupBy gets the Serialized instance through an implicit conversion of the
+   *     // serdes brought into scope through the import DefaultSerdes._ above
+   *     .groupBy((k, v) => v)
+   *
+   *     .count()
+   * }}}
+   *
+   * @param selector a function that computes a new key for grouping
+   * @return a [[KGroupedStream]] that contains the grouped records of the original [[KStream]]
+   * @see `org.apache.kafka.streams.kstream.KStream#groupBy`
+   */ 
+  def groupBy[KR](selector: (K, V) => KR)(implicit serialized: Serialized[KR, V]): KGroupedStream[KR, V] =
+    inner.groupBy(selector.asKeyValueMapper, serialized)
+
+  /**
+   * Join records of this stream with another [[KStream]]'s records using windowed inner equi join with 
+   * serializers and deserializers supplied by the implicit `Joined` instance.
+   *
+   * @param otherStream the [[KStream]] to be joined with this stream
+   * @param joiner      a function that computes the join result for a pair of matching records
+   * @param windows     the specification of the `JoinWindows`
+   * @param joined      an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize 
+   *                    inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   *                    key serde, value serde and other value serde in implicit scope and they will be
+   *                    converted to the instance of `Joined` through implicit conversion
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   * one for each matched record-pair with the same key and within the joining window intervals
+   * @see `org.apache.kafka.streams.kstream.KStream#join`
+   */ 
+  def join[VO, VR](otherStream: KStream[K, VO],
+    joiner: (V, VO) => VR,
+    windows: JoinWindows)(implicit joined: Joined[K, V, VO]): KStream[K, VR] =
+      inner.join[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, joined)
+
+  /**
+   * Join records of this stream with another [[KTable]]'s records using inner equi join with 
+   * serializers and deserializers supplied by the implicit `Joined` instance.
+   *
+   * @param table    the [[KTable]] to be joined with this stream
+   * @param joiner   a function that computes the join result for a pair of matching records
+   * @param joined   an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize 
+   *                 inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   *                 key serde, value serde and other value serde in implicit scope and they will be
+   *                 converted to the instance of `Joined` through implicit conversion
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   * one for each matched record-pair with the same key 
+   * @see `org.apache.kafka.streams.kstream.KStream#join`
+   */ 
+  def join[VT, VR](table: KTable[K, VT],
+    joiner: (V, VT) => VR)(implicit joined: Joined[K, V, VT]): KStream[K, VR] =
+      inner.join[VT, VR](table.inner, joiner.asValueJoiner, joined)
+
+  /**
+   * Join records of this stream with `GlobalKTable`'s records using non-windowed inner equi join.
+   *
+   * @param globalKTable   the `GlobalKTable` to be joined with this stream
+   * @param keyValueMapper a function used to map from the (key, value) of this stream
+   *                       to the key of the `GlobalKTable`
+   * @param joiner         a function that computes the join result for a pair of matching records
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   *                       one output for each input [[KStream]] record
+   * @see `org.apache.kafka.streams.kstream.KStream#join`
+   */ 
+  def join[GK, GV, RV](globalKTable: GlobalKTable[GK, GV],
+    keyValueMapper: (K, V) => GK,
+    joiner: (V, GV) => RV): KStream[K, RV] =
+      inner.join[GK, GV, RV](
+        globalKTable,
+        ((k: K, v: V) => keyValueMapper(k, v)).asKeyValueMapper,
+        ((v: V, gv: GV) => joiner(v, gv)).asValueJoiner
+      )
+
+  /**
+   * Join records of this stream with another [[KStream]]'s records using windowed left equi join with 
+   * serializers and deserializers supplied by the implicit `Joined` instance.
+   *
+   * @param otherStream the [[KStream]] to be joined with this stream
+   * @param joiner      a function that computes the join result for a pair of matching records
+   * @param windows     the specification of the `JoinWindows`
+   * @param joined      an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize 
+   *                    inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   *                    key serde, value serde and other value serde in implicit scope and they will be
+   *                    converted to the instance of `Joined` through implicit conversion
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   *                    one for each matched record-pair with the same key and within the joining window intervals
+   * @see `org.apache.kafka.streams.kstream.KStream#leftJoin`
+   */ 
+  def leftJoin[VO, VR](otherStream: KStream[K, VO],
+    joiner: (V, VO) => VR,
+    windows: JoinWindows)(implicit joined: Joined[K, V, VO]): KStream[K, VR] =
+      inner.leftJoin[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, joined)
+
+  /**
+   * Join records of this stream with another [[KTable]]'s records using left equi join with 
+   * serializers and deserializers supplied by the implicit `Joined` instance.
+   *
+   * @param table    the [[KTable]] to be joined with this stream
+   * @param joiner   a function that computes the join result for a pair of matching records
+   * @param joined   an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize 
+   *                 inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   *                 key serde, value serde and other value serde in implicit scope and they will be
+   *                 converted to the instance of `Joined` through implicit conversion
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   *                 one for each matched record-pair with the same key 
+   * @see `org.apache.kafka.streams.kstream.KStream#leftJoin`
+   */ 
+  def leftJoin[VT, VR](table: KTable[K, VT],
+    joiner: (V, VT) => VR)(implicit joined: Joined[K, V, VT]): KStream[K, VR] =
+      inner.leftJoin[VT, VR](table.inner, joiner.asValueJoiner, joined)
+
+  /**
+   * Join records of this stream with `GlobalKTable`'s records using non-windowed left equi join.
+   *
+   * @param globalKTable   the `GlobalKTable` to be joined with this stream
+   * @param keyValueMapper a function used to map from the (key, value) of this stream
+   *                       to the key of the `GlobalKTable`
+   * @param joiner         a function that computes the join result for a pair of matching records
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   *                       one output for each input [[KStream]] record
+   * @see `org.apache.kafka.streams.kstream.KStream#leftJoin`
+   */ 
+  def leftJoin[GK, GV, RV](globalKTable: GlobalKTable[GK, GV],
+    keyValueMapper: (K, V) => GK,
+    joiner: (V, GV) => RV): KStream[K, RV] = {
+
+    inner.leftJoin[GK, GV, RV](globalKTable, keyValueMapper.asKeyValueMapper, joiner.asValueJoiner)
+  }
+
+  /**
+   * Join records of this stream with another [[KStream]]'s records using windowed outer equi join with 
+   * serializers and deserializers supplied by the implicit `Joined` instance.
+   *
+   * @param otherStream the [[KStream]] to be joined with this stream
+   * @param joiner      a function that computes the join result for a pair of matching records
+   * @param windows     the specification of the `JoinWindows`
+   * @param joined      an implicit `Joined` instance that defines the serdes to be used to serialize/deserialize 
+   *                    inputs and outputs of the joined streams. Instead of `Joined`, the user can also supply
+   *                    key serde, value serde and other value serde in implicit scope and they will be
+   *                    converted to the instance of `Joined` through implicit conversion
+   * @return a [[KStream]] that contains join-records for each key and values computed by the given `joiner`,
+   * one for each matched record-pair with the same key and within the joining window intervals
+   * @see `org.apache.kafka.streams.kstream.KStream#outerJoin`
+   */ 
+  def outerJoin[VO, VR](otherStream: KStream[K, VO],
+    joiner: (V, VO) => VR,
+    windows: JoinWindows)(implicit joined: Joined[K, V, VO]): KStream[K, VR] =
+      inner.outerJoin[VO, VR](otherStream.inner, joiner.asValueJoiner, windows, joined)
+
+  /**
+   * Merge this stream and the given stream into one larger stream.
+   * 
    
+   * There is no ordering guarantee between records from this `KStream` and records from the provided `KStream` 
+   * in the merged stream. Relative order is preserved within each input stream though (ie, records within 
+   * one input stream are processed in order).
+   *
+   * @param stream a stream which is to be merged into this stream
+   * @return a merged stream containing all records from this and the provided [[KStream]]
+   * @see `org.apache.kafka.streams.kstream.KStream#merge`
+   */
+  def merge(stream: KStream[K, V]): KStream[K, V] = inner.merge(stream.inner)
+
+  /**
+   * Perform an action on each record of {@code KStream}.
+   * 
    
+   * Peek is a non-terminal operation that triggers a side effect (such as logging or statistics collection)
+   * and returns an unchanged stream.
+   *
+   * @param action an action to perform on each record
+   * @see `org.apache.kafka.streams.kstream.KStream#peek`
+   */
+  def peek(action: (K, V) => Unit): KStream[K, V] = {
+    inner.peek((k: K, v: V) => action(k, v))
+  }
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
new file mode 100644
index 0000000000000..0369ee5aa589d
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
@@ -0,0 +1,292 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+package kstream
+
+import org.apache.kafka.streams.kstream.{KTable => KTableJ, _}
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.FunctionConversions._
+
+/**
+ * Wraps the Java class [[org.apache.kafka.streams.kstream.KTable]] and delegates method calls to the underlying Java object.
+ *
+ * @param [K] Type of keys
+ * @param [V] Type of values
+ * @param inner The underlying Java abstraction for KTable
+ *
+ * @see `org.apache.kafka.streams.kstream.KTable`
+ */
+class KTable[K, V](val inner: KTableJ[K, V]) {
+
+  /**
+   * Create a new [[KTable]] that consists all records of this [[KTable]] which satisfies the given
+   * predicate
+   *
+   * @param predicate a filter that is applied to each record
+   * @return a [[KTable]] that contains only those records that satisfy the given predicate
+   * @see `org.apache.kafka.streams.kstream.KTable#filter`
+   */ 
+  def filter(predicate: (K, V) => Boolean): KTable[K, V] = {
+    inner.filter(predicate(_, _))
+  }
+
+  /**
+   * Create a new [[KTable]] that consists all records of this [[KTable]] which satisfies the given
+   * predicate
+   *
+   * @param predicate a filter that is applied to each record
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains only those records that satisfy the given predicate
+   * @see `org.apache.kafka.streams.kstream.KTable#filter`
+   */ 
+  def filter(predicate: (K, V) => Boolean,
+    materialized: Materialized[K, V, ByteArrayKeyValueStore]): KTable[K, V] = {
+    inner.filter(predicate.asPredicate, materialized)
+  }
+
+  /**
+   * Create a new [[KTable]] that consists all records of this [[KTable]] which do not satisfy the given
+   * predicate
+   *
+   * @param predicate a filter that is applied to each record
+   * @return a [[KTable]] that contains only those records that do not satisfy the given predicate
+   * @see `org.apache.kafka.streams.kstream.KTable#filterNot`
+   */ 
+  def filterNot(predicate: (K, V) => Boolean): KTable[K, V] = {
+    inner.filterNot(predicate(_, _))
+  }
+
+  /**
+   * Create a new [[KTable]] that consists all records of this [[KTable]] which do not satisfy the given
+   * predicate
+   *
+   * @param predicate a filter that is applied to each record
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains only those records that do not satisfy the given predicate
+   * @see `org.apache.kafka.streams.kstream.KTable#filterNot`
+   */ 
+  def filterNot(predicate: (K, V) => Boolean,
+    materialized: Materialized[K, V, ByteArrayKeyValueStore]): KTable[K, V] = {
+    inner.filterNot(predicate.asPredicate, materialized)
+  }
+
+  /**
+   * Create a new [[KTable]] by transforming the value of each record in this [[KTable]] into a new value
+   * (with possible new type) in the new [[KTable]].
+   * 
    
+   * The provided `mapper`, a function `V => VR` is applied to each input record value and computes a new value for it
+   *
+   * @param mapper, a function `V => VR` that computes a new output value
+   * @return a [[KTable]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KTable#mapValues`
+   */ 
+  def mapValues[VR](mapper: V => VR): KTable[K, VR] = {
+    inner.mapValues[VR](mapper.asValueMapper)
+  }
+
+  /**
+   * Create a new [[KTable]] by transforming the value of each record in this [[KTable]] into a new value
+   * (with possible new type) in the new [[KTable]].
+   * 
    
+   * The provided `mapper`, a function `V => VR` is applied to each input record value and computes a new value for it
+   *
+   * @param mapper, a function `V => VR` that computes a new output value
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KTable#mapValues`
+   */ 
+  def mapValues[VR](mapper: V => VR,
+    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+    inner.mapValues[VR](mapper.asValueMapper, materialized)
+  }
+
+  /**
+   * Create a new [[KTable]] by transforming the value of each record in this [[KTable]] into a new value
+   * (with possible new type) in the new [[KTable]].
+   * 
    
+   * The provided `mapper`, a function `(K, V) => VR` is applied to each input record value and computes a new value for it
+   *
+   * @param mapper, a function `(K, V) => VR` that computes a new output value
+   * @return a [[KTable]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KTable#mapValues`
+   */ 
+  def mapValues[VR](mapper: (K, V) => VR): KTable[K, VR] = {
+    inner.mapValues[VR](mapper.asValueMapperWithKey)
+  }
+
+  /**
+   * Create a new [[KTable]] by transforming the value of each record in this [[KTable]] into a new value
+   * (with possible new type) in the new [[KTable]].
+   * 
    
+   * The provided `mapper`, a function `(K, V) => VR` is applied to each input record value and computes a new value for it
+   *
+   * @param mapper, a function `(K, V) => VR` that computes a new output value
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains records with unmodified key and new values (possibly of different type)
+   * @see `org.apache.kafka.streams.kstream.KTable#mapValues`
+   */ 
+  def mapValues[VR](mapper: (K, V) => VR,
+    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+    inner.mapValues[VR](mapper.asValueMapperWithKey)
+  }
+
+  /**
+   * Convert this changelog stream to a [[KStream]].
+   *
+   * @return a [[KStream]] that contains the same records as this [[KTable]]
+   * @see `org.apache.kafka.streams.kstream.KTable#toStream`
+   */
+  def toStream: KStream[K, V] = inner.toStream
+
+  /**
+   * Convert this changelog stream to a [[KStream]] using the given key/value mapper to select the new key
+   *
+   * @param mapper a function that computes a new key for each record
+   * @return a [[KStream]] that contains the same records as this [[KTable]]
+   * @see `org.apache.kafka.streams.kstream.KTable#toStream`
+   */
+  def toStream[KR](mapper: (K, V) => KR): KStream[KR, V] = {
+    inner.toStream[KR](mapper.asKeyValueMapper)
+  }
+
+  /**
+   * Re-groups the records of this [[KTable]] using the provided key/value mapper
+   * and `Serde`s as specified by `Serialized`.
+   *
+   * @param selector      a function that computes a new grouping key and value to be aggregated
+   * @param serialized    the `Serialized` instance used to specify `Serdes`
+   * @return a [[KGroupedTable]] that contains the re-grouped records of the original [[KTable]]
+   * @see `org.apache.kafka.streams.kstream.KTable#groupBy`
+   */ 
+  def groupBy[KR, VR](selector: (K, V) => (KR, VR))(implicit serialized: Serialized[KR, VR]): KGroupedTable[KR, VR] = {
+    inner.groupBy(selector.asKeyValueMapper, serialized)
+  }
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner equi join.
+   *
+   * @param other  the other [[KTable]] to be joined with this [[KTable]]
+   * @param joiner a function that computes the join result for a pair of matching records
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   * @see `org.apache.kafka.streams.kstream.KTable#join`
+   */ 
+  def join[VO, VR](other: KTable[K, VO],
+    joiner: (V, VO) => VR): KTable[K, VR] = {
+
+    inner.join[VO, VR](other.inner, joiner.asValueJoiner)
+  }
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed inner equi join.
+   *
+   * @param other  the other [[KTable]] to be joined with this [[KTable]]
+   * @param joiner a function that computes the join result for a pair of matching records
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   * @see `org.apache.kafka.streams.kstream.KTable#join`
+   */ 
+  def join[VO, VR](other: KTable[K, VO],
+    joiner: (V, VO) => VR,
+    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+
+    inner.join[VO, VR](other.inner, joiner.asValueJoiner, materialized)
+  }
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left equi join.
+   *
+   * @param other  the other [[KTable]] to be joined with this [[KTable]]
+   * @param joiner a function that computes the join result for a pair of matching records
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   * @see `org.apache.kafka.streams.kstream.KTable#leftJoin`
+   */ 
+  def leftJoin[VO, VR](other: KTable[K, VO],
+    joiner: (V, VO) => VR): KTable[K, VR] = {
+
+    inner.leftJoin[VO, VR](other.inner, joiner.asValueJoiner)
+  }
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed left equi join.
+   *
+   * @param other  the other [[KTable]] to be joined with this [[KTable]]
+   * @param joiner a function that computes the join result for a pair of matching records
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   * @see `org.apache.kafka.streams.kstream.KTable#leftJoin`
+   */ 
+  def leftJoin[VO, VR](other: KTable[K, VO],
+    joiner: (V, VO) => VR,
+    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+
+    inner.leftJoin[VO, VR](other.inner, joiner.asValueJoiner, materialized)
+  }
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed outer equi join.
+   *
+   * @param other  the other [[KTable]] to be joined with this [[KTable]]
+   * @param joiner a function that computes the join result for a pair of matching records
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   * @see `org.apache.kafka.streams.kstream.KTable#leftJoin`
+   */ 
+  def outerJoin[VO, VR](other: KTable[K, VO],
+    joiner: (V, VO) => VR): KTable[K, VR] = {
+
+    inner.outerJoin[VO, VR](other.inner, joiner.asValueJoiner)
+  }
+
+  /**
+   * Join records of this [[KTable]] with another [[KTable]]'s records using non-windowed outer equi join.
+   *
+   * @param other  the other [[KTable]] to be joined with this [[KTable]]
+   * @param joiner a function that computes the join result for a pair of matching records
+   * @param materialized  a `Materialized` that describes how the `StateStore` for the resulting [[KTable]]
+   *                      should be materialized. 
+   * @return a [[KTable]] that contains join-records for each key and values computed by the given joiner,
+   * one for each matched record-pair with the same key
+   * @see `org.apache.kafka.streams.kstream.KTable#leftJoin`
+   */ 
+  def outerJoin[VO, VR](other: KTable[K, VO],
+    joiner: (V, VO) => VR,
+    materialized: Materialized[K, VR, ByteArrayKeyValueStore]): KTable[K, VR] = {
+
+    inner.outerJoin[VO, VR](other.inner, joiner.asValueJoiner, materialized)
+  }
+
+  /**
+   * Get the name of the local state store used that can be used to query this [[KTable]].
+   *
+   * @return the underlying state store name, or `null` if this [[KTable]] cannot be queried.
+   */
+  def queryableStoreName: String =
+    inner.queryableStoreName
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/SessionWindowedKStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/SessionWindowedKStream.scala
new file mode 100644
index 0000000000000..7e9fa0713fe29
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/SessionWindowedKStream.scala
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+package kstream
+
+import org.apache.kafka.streams.kstream.{SessionWindowedKStream => SessionWindowedKStreamJ, _}
+import org.apache.kafka.streams.state.SessionStore
+import org.apache.kafka.common.utils.Bytes
+
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.FunctionConversions._
+
+/**
+ * Wraps the Java class SessionWindowedKStream and delegates method calls to the underlying Java object.
+ *
+ * @param [K] Type of keys
+ * @param [V] Type of values
+ * @param inner The underlying Java abstraction for SessionWindowedKStream
+ *
+ * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream`
+ */
+class SessionWindowedKStream[K, V](val inner: SessionWindowedKStreamJ[K, V]) {
+
+  /**
+   * Aggregate the values of records in this stream by the grouped key and defined `SessionWindows`.
+   *
+   * @param initializer    the initializer function
+   * @param aggregator     the aggregator function
+   * @param sessionMerger  the merger function
+   * @return a windowed [[KTable]] that contains "update" records with unmodified keys, and values that represent
+   * the latest (rolling) aggregate for each key within a window
+   * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream#aggregate`
+   */
+  def aggregate[VR](initializer: () => VR,
+    aggregator: (K, V, VR) => VR,
+    merger: (K, VR, VR) => VR): KTable[Windowed[K], VR] = {
+
+    inner.aggregate(initializer.asInitializer, aggregator.asAggregator, merger.asMerger)
+  }
+
+  /**
+   * Aggregate the values of records in this stream by the grouped key and defined `SessionWindows`.
+   *
+   * @param initializer    the initializer function
+   * @param aggregator     the aggregator function
+   * @param sessionMerger  the merger function
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a windowed [[KTable]] that contains "update" records with unmodified keys, and values that represent
+   * the latest (rolling) aggregate for each key within a window
+   * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream#aggregate`
+   */
+  def aggregate[VR](initializer: () => VR,
+    aggregator: (K, V, VR) => VR,
+    merger: (K, VR, VR) => VR,
+    materialized: Materialized[K, VR, SessionStore[Bytes, Array[Byte]]]): KTable[Windowed[K], VR] = {
+
+    inner.aggregate(initializer.asInitializer, aggregator.asAggregator, merger.asMerger, materialized)
+  }
+
+  /**
+   * Count the number of records in this stream by the grouped key into `SessionWindows`.
+   *
+   * @return a windowed [[KTable]] that contains "update" records with unmodified keys and `Long` values
+   * that represent the latest (rolling) count (i.e., number of records) for each key within a window
+   * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream#count`
+   */
+  def count(): KTable[Windowed[K], Long] = {
+    val c: KTable[Windowed[K], java.lang.Long] = inner.count()
+    c.mapValues[Long](Long2long(_))
+  }
+
+  /**
+   * Count the number of records in this stream by the grouped key into `SessionWindows`.
+   *
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a windowed [[KTable]] that contains "update" records with unmodified keys and `Long` values
+   * that represent the latest (rolling) count (i.e., number of records) for each key within a window
+   * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream#count`
+   */
+  def count(materialized: Materialized[K, Long, SessionStore[Bytes, Array[Byte]]]): KTable[Windowed[K], Long] =
+    inner.count(materialized)
+
+  /**
+   * Combine values of this stream by the grouped key into {@link SessionWindows}.
+   *
+   * @param reducer           a reducer function that computes a new aggregate result. 
+   * @return a windowed [[KTable]] that contains "update" records with unmodified keys, and values that represent
+   * the latest (rolling) aggregate for each key within a window
+   * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream#reduce`
+   */
+  def reduce(reducer: (V, V) => V): KTable[Windowed[K], V] = {
+    inner.reduce((v1, v2) => reducer(v1, v2))
+  }
+
+  /**
+   * Combine values of this stream by the grouped key into {@link SessionWindows}.
+   *
+   * @param reducer           a reducer function that computes a new aggregate result. 
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a windowed [[KTable]] that contains "update" records with unmodified keys, and values that represent
+   * the latest (rolling) aggregate for each key within a window
+   * @see `org.apache.kafka.streams.kstream.SessionWindowedKStream#reduce`
+   */
+  def reduce(reducer: (V, V) => V,
+    materialized: Materialized[K, V, SessionStore[Bytes, Array[Byte]]]): KTable[Windowed[K], V] = {
+    inner.reduce(reducer.asReducer, materialized)
+  }
+}
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/TimeWindowedKStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/TimeWindowedKStream.scala
new file mode 100644
index 0000000000000..1aa19786d5ab2
--- /dev/null
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/TimeWindowedKStream.scala
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+package kstream
+
+import org.apache.kafka.streams.kstream.{TimeWindowedKStream => TimeWindowedKStreamJ, _}
+import org.apache.kafka.streams.state.WindowStore
+import org.apache.kafka.common.utils.Bytes
+import org.apache.kafka.common.serialization.Serde
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.FunctionConversions._
+
+/**
+ * Wraps the Java class TimeWindowedKStream and delegates method calls to the underlying Java object.
+ *
+ * @param [K] Type of keys
+ * @param [V] Type of values
+ * @param inner The underlying Java abstraction for TimeWindowedKStream
+ *
+ * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream`
+ */
+class TimeWindowedKStream[K, V](val inner: TimeWindowedKStreamJ[K, V]) {
+
+  /**
+   * Aggregate the values of records in this stream by the grouped key.
+   *
+   * @param initializer   an initializer function that computes an initial intermediate aggregation result
+   * @param aggregator    an aggregator function that computes a new aggregate result
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream#aggregate`
+   */
+  def aggregate[VR](initializer: () => VR,
+    aggregator: (K, V, VR) => VR): KTable[Windowed[K], VR] = {
+
+    inner.aggregate(initializer.asInitializer, aggregator.asAggregator)
+  }
+
+  /**
+   * Aggregate the values of records in this stream by the grouped key.
+   *
+   * @param initializer   an initializer function that computes an initial intermediate aggregation result
+   * @param aggregator    an aggregator function that computes a new aggregate result
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream#aggregate`
+   */
+  def aggregate[VR](initializer: () => VR,
+    aggregator: (K, V, VR) => VR,
+    materialized: Materialized[K, VR, WindowStore[Bytes, Array[Byte]]]): KTable[Windowed[K], VR] = {
+
+    inner.aggregate(initializer.asInitializer, aggregator.asAggregator, materialized)
+  }
+
+  /**
+   * Count the number of records in this stream by the grouped key and the defined windows.
+   *
+   * @return a [[KTable]] that contains "update" records with unmodified keys and `Long` values that
+   * represent the latest (rolling) count (i.e., number of records) for each key
+   * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream#count`
+   */ 
+  def count(): KTable[Windowed[K], Long] = {
+    val c: KTable[Windowed[K], java.lang.Long] = inner.count()
+    c.mapValues[Long](Long2long(_))
+  }
+
+  /**
+   * Count the number of records in this stream by the grouped key and the defined windows.
+   *
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys and `Long` values that
+   * represent the latest (rolling) count (i.e., number of records) for each key
+   * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream#count`
+   */ 
+  def count(materialized: Materialized[K, Long, WindowStore[Bytes, Array[Byte]]]): KTable[Windowed[K], Long] = {
+    val c: KTable[Windowed[K], java.lang.Long] = 
+      inner.count(materialized.asInstanceOf[Materialized[K, java.lang.Long, WindowStore[Bytes, Array[Byte]]]])
+    c.mapValues[Long](Long2long(_))
+  }
+
+  /**
+   * Combine the values of records in this stream by the grouped key.
+   *
+   * @param reducer   a function that computes a new aggregate result
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream#reduce`
+   */
+  def reduce(reducer: (V, V) => V): KTable[Windowed[K], V] = {
+    inner.reduce(reducer.asReducer)
+  }
+
+  /**
+   * Combine the values of records in this stream by the grouped key.
+   *
+   * @param reducer   a function that computes a new aggregate result
+   * @param materialized  an instance of `Materialized` used to materialize a state store. 
+   * @return a [[KTable]] that contains "update" records with unmodified keys, and values that represent the
+   * latest (rolling) aggregate for each key
+   * @see `org.apache.kafka.streams.kstream.TimeWindowedKStream#reduce`
+   */
+  def reduce(reducer: (V, V) => V,
+    materialized: Materialized[K, V, WindowStore[Bytes, Array[Byte]]]): KTable[Windowed[K], V] = {
+
+    inner.reduce(reducer.asReducer, materialized)
+  }
+}
diff --git a/streams/streams-scala/src/test/resources/log4j.properties b/streams/streams-scala/src/test/resources/log4j.properties
new file mode 100644
index 0000000000000..93ffc165654a2
--- /dev/null
+++ b/streams/streams-scala/src/test/resources/log4j.properties
@@ -0,0 +1,34 @@
+#   Copyright (C) 2018 Lightbend Inc. 
+#   Copyright (C) 2017-2018 Alexis Seigneurin.
+#  
+#   Licensed to the Apache Software Foundation (ASF) under one or more
+#   contributor license agreements. See the NOTICE file distributed with
+#   this work for additional information regarding copyright ownership.
+#   The ASF licenses this file to You under the Apache License, Version 2.0
+#   (the "License"); you may not use this file except in compliance with
+#   the License. You may obtain a copy of the License at
+#  
+#      http://www.apache.org/licenses/LICENSE-2.0
+#  
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=INFO, R
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.File=logs/kafka-streams-scala.log
+
+log4j.appender.R.MaxFileSize=100KB
+# Keep one backup file
+log4j.appender.R.MaxBackupIndex=1
+
+# A1 uses PatternLayout.
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala
new file mode 100644
index 0000000000000..24974c40f7283
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala
@@ -0,0 +1,237 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import java.util.Properties
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.Assert._
+import org.junit.rules.TemporaryFolder
+import org.junit._
+
+import org.apache.kafka.streams.integration.utils.{EmbeddedKafkaCluster, IntegrationTestUtils}
+import org.apache.kafka.clients.consumer.ConsumerConfig
+import org.apache.kafka.clients.producer.ProducerConfig
+
+import org.apache.kafka.common.serialization._
+import org.apache.kafka.common.utils.MockTime
+import org.apache.kafka.test.TestUtils
+import org.apache.kafka.streams._
+import org.apache.kafka.streams.scala.kstream._
+
+import ImplicitConversions._
+import com.typesafe.scalalogging.LazyLogging
+
+/**
+ * Test suite that does an example to demonstrate stream-table joins in Kafka Streams
+ * 
    
+ * The suite contains the test case using Scala APIs `testShouldCountClicksPerRegion` and the same test case using the
+ * Java APIs `testShouldCountClicksPerRegionJava`. The idea is to demonstrate that both generate the same result.
+ * 
    
+ * Note: In the current project settings SAM type conversion is turned off as it's experimental in Scala 2.11.
+ * Hence the native Java API based version is more verbose.
+ */ 
+class StreamToTableJoinScalaIntegrationTestImplicitSerdes extends JUnitSuite
+  with StreamToTableJoinTestData with LazyLogging {
+
+  private val privateCluster: EmbeddedKafkaCluster = new EmbeddedKafkaCluster(1)
+
+  @Rule def cluster: EmbeddedKafkaCluster = privateCluster
+
+  final val alignedTime = (System.currentTimeMillis() / 1000 + 1) * 1000
+  val mockTime: MockTime = cluster.time
+  mockTime.setCurrentTimeMs(alignedTime)
+
+  val tFolder: TemporaryFolder = new TemporaryFolder(TestUtils.tempDirectory())
+  @Rule def testFolder: TemporaryFolder = tFolder
+
+  @Before
+  def startKafkaCluster(): Unit = {
+    cluster.createTopic(userClicksTopic)
+    cluster.createTopic(userRegionsTopic)
+    cluster.createTopic(outputTopic)
+    cluster.createTopic(userClicksTopicJ)
+    cluster.createTopic(userRegionsTopicJ)
+    cluster.createTopic(outputTopicJ)
+  }
+
+  @Test def testShouldCountClicksPerRegion(): Unit = {
+
+    // DefaultSerdes brings into scope implicit serdes (mostly for primitives) that will set up all Serialized, Produced, 
+    // Consumed and Joined instances. So all APIs below that accept Serialized, Produced, Consumed or Joined will
+    // get these instances automatically
+    import DefaultSerdes._
+
+    val streamsConfiguration: Properties = getStreamsConfiguration()
+
+    val builder = new StreamsBuilder()
+
+    val userClicksStream: KStream[String, Long] = builder.stream(userClicksTopic)
+
+    val userRegionsTable: KTable[String, String] = builder.table(userRegionsTopic)
+
+    // Compute the total per region by summing the individual click counts per region.
+    val clicksPerRegion: KTable[String, Long] =
+      userClicksStream
+
+        // Join the stream against the table.
+        .leftJoin(userRegionsTable, (clicks: Long, region: String) => (if (region == null) "UNKNOWN" else region, clicks))
+
+        // Change the stream from  ->  to  -> 
+        .map((_, regionWithClicks) => regionWithClicks)
+
+        // Compute the total per region by summing the individual click counts per region.
+        .groupByKey
+        .reduce(_ + _)
+
+    // Write the (continuously updating) results to the output topic.
+    clicksPerRegion.toStream.to(outputTopic)
+
+    val streams: KafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration)
+    streams.start()
+
+
+    val actualClicksPerRegion: java.util.List[KeyValue[String, Long]] = 
+      produceNConsume(userClicksTopic, userRegionsTopic, outputTopic)
+
+    streams.close()
+
+    import collection.JavaConverters._
+    assertEquals(actualClicksPerRegion.asScala.sortBy(_.key), expectedClicksPerRegion.sortBy(_.key))
+  }
+
+  @Test def testShouldCountClicksPerRegionJava(): Unit = {
+
+    import org.apache.kafka.streams.{KafkaStreams => KafkaStreamsJ, StreamsBuilder => StreamsBuilderJ}
+    import org.apache.kafka.streams.kstream.{KTable => KTableJ, KStream => KStreamJ, KGroupedStream => KGroupedStreamJ, _}
+    import collection.JavaConverters._
+    import java.lang.{Long => JLong}
+
+    val streamsConfiguration: Properties = getStreamsConfiguration()
+
+    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName())
+    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName())
+
+    val builder: StreamsBuilderJ = new StreamsBuilderJ()
+
+    val userClicksStream: KStreamJ[String, JLong] = 
+      builder.stream[String, JLong](userClicksTopicJ, Consumed.`with`(Serdes.String(), Serdes.Long()))
+
+    val userRegionsTable: KTableJ[String, String] = 
+      builder.table[String, String](userRegionsTopicJ, Consumed.`with`(Serdes.String(), Serdes.String()))
+
+    // Join the stream against the table.
+    val userClicksJoinRegion: KStreamJ[String, (String, JLong)] = userClicksStream
+      .leftJoin(userRegionsTable, 
+        new ValueJoiner[JLong, String, (String, JLong)] {
+          def apply(clicks: JLong, region: String): (String, JLong) = 
+            (if (region == null) "UNKNOWN" else region, clicks)
+        }, 
+        Joined.`with`[String, JLong, String](Serdes.String(), Serdes.Long(), Serdes.String())) 
+
+    // Change the stream from  ->  to  -> 
+    val clicksByRegion : KStreamJ[String, JLong] = userClicksJoinRegion
+      .map { 
+        new KeyValueMapper[String, (String, JLong), KeyValue[String, JLong]] {
+          def apply(k: String, regionWithClicks: (String, JLong)) = new KeyValue[String, JLong](regionWithClicks._1, regionWithClicks._2)
+        }
+      }
+        
+    // Compute the total per region by summing the individual click counts per region.
+    val clicksPerRegion: KTableJ[String, JLong] = clicksByRegion
+      .groupByKey(Serialized.`with`(Serdes.String(), Serdes.Long()))
+      .reduce {
+        new Reducer[JLong] {
+          def apply(v1: JLong, v2: JLong) = v1 + v2
+        }
+      }
+        
+    // Write the (continuously updating) results to the output topic.
+    clicksPerRegion.toStream.to(outputTopicJ, Produced.`with`(Serdes.String(), Serdes.Long()))
+
+    val streams: KafkaStreamsJ = new KafkaStreamsJ(builder.build(), streamsConfiguration)
+
+    streams.start()
+
+    val actualClicksPerRegion: java.util.List[KeyValue[String, Long]] = 
+      produceNConsume(userClicksTopicJ, userRegionsTopicJ, outputTopicJ)
+
+    streams.close()
+    assertEquals(actualClicksPerRegion.asScala.sortBy(_.key), expectedClicksPerRegion.sortBy(_.key))
+  }
+
+  private def getStreamsConfiguration(): Properties = {
+    val streamsConfiguration: Properties = new Properties()
+
+    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "stream-table-join-scala-integration-test")
+    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, "10000")
+    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, testFolder.getRoot().getPath())
+
+    streamsConfiguration
+  }
+
+  private def getUserRegionsProducerConfig(): Properties = {
+    val p = new Properties()
+    p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    p.put(ProducerConfig.ACKS_CONFIG, "all")
+    p.put(ProducerConfig.RETRIES_CONFIG, "0")
+    p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
+    p.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
+    p
+  }
+
+  private def getUserClicksProducerConfig(): Properties = {
+    val p = new Properties()
+    p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    p.put(ProducerConfig.ACKS_CONFIG, "all")
+    p.put(ProducerConfig.RETRIES_CONFIG, "0")
+    p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
+    p.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[LongSerializer])
+    p
+  }
+
+  private def getConsumerConfig(): Properties = {
+    val p = new Properties()
+    p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    p.put(ConsumerConfig.GROUP_ID_CONFIG, "join-scala-integration-test-standard-consumer")
+    p.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer])
+    p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[LongDeserializer])
+    p
+  }
+
+  private def produceNConsume(userClicksTopic: String, userRegionsTopic: String, outputTopic: String): java.util.List[KeyValue[String, Long]] = {
+
+    import collection.JavaConverters._
+    
+    // Publish user-region information.
+    val userRegionsProducerConfig: Properties = getUserRegionsProducerConfig()
+    IntegrationTestUtils.produceKeyValuesSynchronously(userRegionsTopic, userRegions.asJava, userRegionsProducerConfig, mockTime, false)
+
+    // Publish user-click information.
+    val userClicksProducerConfig: Properties = getUserClicksProducerConfig()
+    IntegrationTestUtils.produceKeyValuesSynchronously(userClicksTopic, userClicks.asJava, userClicksProducerConfig, mockTime, false)
+
+    // consume and verify result
+    val consumerConfig = getConsumerConfig()
+
+    IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(consumerConfig, outputTopic, expectedClicksPerRegion.size)
+  }
+}
+
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinTestData.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinTestData.scala
new file mode 100644
index 0000000000000..45715a7abe6a5
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinTestData.scala
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import org.apache.kafka.streams.KeyValue
+
+trait StreamToTableJoinTestData {
+  val brokers = "localhost:9092"
+
+  val userClicksTopic = s"user-clicks"
+  val userRegionsTopic = s"user-regions"
+  val outputTopic = s"output-topic"
+
+  val userClicksTopicJ = s"user-clicks-j"
+  val userRegionsTopicJ = s"user-regions-j"
+  val outputTopicJ = s"output-topic-j"
+
+  // Input 1: Clicks per user (multiple records allowed per user).
+  val userClicks: Seq[KeyValue[String, Long]] = Seq(
+    new KeyValue("alice", 13L),
+    new KeyValue("bob", 4L),
+    new KeyValue("chao", 25L),
+    new KeyValue("bob", 19L),
+    new KeyValue("dave", 56L),
+    new KeyValue("eve", 78L),
+    new KeyValue("alice", 40L),
+    new KeyValue("fang", 99L)
+  )
+
+  // Input 2: Region per user (multiple records allowed per user).
+  val userRegions: Seq[KeyValue[String, String]] = Seq(
+    new KeyValue("alice", "asia"), /* Alice lived in Asia originally... */
+    new KeyValue("bob", "americas"),
+    new KeyValue("chao", "asia"),
+    new KeyValue("dave", "europe"),
+    new KeyValue("alice", "europe"), /* ...but moved to Europe some time later. */
+    new KeyValue("eve", "americas"),
+    new KeyValue("fang", "asia")
+  )
+
+  val expectedClicksPerRegion: Seq[KeyValue[String, Long]] = Seq(
+    new KeyValue("americas", 101L),
+    new KeyValue("europe", 109L),
+    new KeyValue("asia", 124L)
+  )
+}
+
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
new file mode 100644
index 0000000000000..89b2935b9566e
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/TopologyTest.scala
@@ -0,0 +1,199 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import java.util.Properties
+import java.util.regex.Pattern
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.Assert._
+import org.junit._
+
+import org.apache.kafka.streams.scala.kstream._
+
+import org.apache.kafka.common.serialization._
+
+import ImplicitConversions._
+import com.typesafe.scalalogging.LazyLogging
+
+import org.apache.kafka.streams.{KafkaStreams => KafkaStreamsJ, StreamsBuilder => StreamsBuilderJ, _}
+import org.apache.kafka.streams.kstream.{KTable => KTableJ, KStream => KStreamJ, KGroupedStream => KGroupedStreamJ, _}
+import collection.JavaConverters._
+
+/**
+ * Test suite that verifies that the topology built by the Java and Scala APIs match.
+ */ 
+class TopologyTest extends JUnitSuite with LazyLogging {
+
+  val inputTopic = "input-topic"
+  val userClicksTopic = "user-clicks-topic"
+  val userRegionsTopic = "user-regions-topic"
+
+  val pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS)
+
+  @Test def shouldBuildIdenticalTopologyInJavaNScalaSimple() = {
+
+    // build the Scala topology
+    def getTopologyScala(): TopologyDescription = {
+
+      import DefaultSerdes._
+      import collection.JavaConverters._
+  
+      val streamBuilder = new StreamsBuilder
+      val textLines = streamBuilder.stream[String, String](inputTopic)
+  
+      val _: KStream[String, String] =
+        textLines.flatMapValues(v => pattern.split(v.toLowerCase))
+  
+      streamBuilder.build().describe()
+    }
+  
+    // build the Java topology
+    def getTopologyJava(): TopologyDescription = {
+
+      val streamBuilder = new StreamsBuilderJ
+      val textLines = streamBuilder.stream[String, String](inputTopic)
+  
+      val _: KStreamJ[String, String] = textLines.flatMapValues {
+        new ValueMapper[String, java.lang.Iterable[String]] {
+          def apply(s: String): java.lang.Iterable[String] = pattern.split(s.toLowerCase).toIterable.asJava
+        }
+      }
+      streamBuilder.build().describe()
+    }
+
+    // should match
+    assertEquals(getTopologyScala(), getTopologyJava())
+  }
+
+  @Test def shouldBuildIdenticalTopologyInJavaNScalaAggregate() = {
+
+    // build the Scala topology
+    def getTopologyScala(): TopologyDescription = {
+
+      import DefaultSerdes._
+      import collection.JavaConverters._
+  
+      val streamBuilder = new StreamsBuilder
+      val textLines = streamBuilder.stream[String, String](inputTopic)
+  
+      val _: KTable[String, Long] =
+        textLines.flatMapValues(v => pattern.split(v.toLowerCase))
+          .groupBy((k, v) => v)
+          .count()
+  
+      streamBuilder.build().describe()
+    }
+
+    // build the Java topology
+    def getTopologyJava(): TopologyDescription = {
+
+      val streamBuilder = new StreamsBuilderJ
+      val textLines: KStreamJ[String, String] = streamBuilder.stream[String, String](inputTopic)
+  
+      val splits: KStreamJ[String, String] = textLines.flatMapValues {
+        new ValueMapper[String, java.lang.Iterable[String]] {
+          def apply(s: String): java.lang.Iterable[String] = pattern.split(s.toLowerCase).toIterable.asJava
+        }
+      }
+  
+      val grouped: KGroupedStreamJ[String, String] = splits.groupBy {
+        new KeyValueMapper[String, String, String] {
+          def apply(k: String, v: String): String = v
+        }
+      }
+  
+      val wordCounts: KTableJ[String, java.lang.Long] = grouped.count()
+  
+      streamBuilder.build().describe()
+    }
+
+    // should match
+    assertEquals(getTopologyScala(), getTopologyJava())
+  }
+
+  @Test def shouldBuildIdenticalTopologyInJavaNScalaJoin() = {
+
+    // build the Scala topology
+    def getTopologyScala(): TopologyDescription = {
+      import DefaultSerdes._
+  
+      val builder = new StreamsBuilder()
+  
+      val userClicksStream: KStream[String, Long] = builder.stream(userClicksTopic)
+  
+      val userRegionsTable: KTable[String, String] = builder.table(userRegionsTopic)
+  
+      val clicksPerRegion: KTable[String, Long] =
+        userClicksStream
+          .leftJoin(userRegionsTable, (clicks: Long, region: String) => (if (region == null) "UNKNOWN" else region, clicks))
+          .map((_, regionWithClicks) => regionWithClicks)
+          .groupByKey
+          .reduce(_ + _)
+
+      builder.build().describe()
+    }
+
+    // build the Java topology
+    def getTopologyJava(): TopologyDescription = {
+
+      import java.lang.{Long => JLong}
+  
+      val builder: StreamsBuilderJ = new StreamsBuilderJ()
+  
+      val userClicksStream: KStreamJ[String, JLong] = 
+        builder.stream[String, JLong](userClicksTopic, Consumed.`with`(Serdes.String(), Serdes.Long()))
+  
+      val userRegionsTable: KTableJ[String, String] = 
+        builder.table[String, String](userRegionsTopic, Consumed.`with`(Serdes.String(), Serdes.String()))
+  
+      // Join the stream against the table.
+      val userClicksJoinRegion: KStreamJ[String, (String, JLong)] = userClicksStream
+        .leftJoin(userRegionsTable, 
+          new ValueJoiner[JLong, String, (String, JLong)] {
+            def apply(clicks: JLong, region: String): (String, JLong) = 
+              (if (region == null) "UNKNOWN" else region, clicks)
+          }, 
+          Joined.`with`[String, JLong, String](Serdes.String(), Serdes.Long(), Serdes.String())) 
+  
+      // Change the stream from  ->  to  -> 
+      val clicksByRegion : KStreamJ[String, JLong] = userClicksJoinRegion
+        .map { 
+          new KeyValueMapper[String, (String, JLong), KeyValue[String, JLong]] {
+            def apply(k: String, regionWithClicks: (String, JLong)) = new KeyValue[String, JLong](regionWithClicks._1, regionWithClicks._2)
+          }
+        }
+          
+      // Compute the total per region by summing the individual click counts per region.
+      val clicksPerRegion: KTableJ[String, JLong] = clicksByRegion
+        .groupByKey(Serialized.`with`(Serdes.String(), Serdes.Long()))
+        .reduce {
+          new Reducer[JLong] {
+            def apply(v1: JLong, v2: JLong) = v1 + v2
+          }
+        }
+
+      builder.build().describe()
+    }
+
+    // should match
+    assertEquals(getTopologyScala(), getTopologyJava())
+  }
+}
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/WordCountTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/WordCountTest.scala
new file mode 100644
index 0000000000000..f71e0cb4a1374
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/WordCountTest.scala
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2018 Lightbend Inc. 
+ * Copyright (C) 2017-2018 Alexis Seigneurin.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import java.util.Properties
+import java.util.regex.Pattern
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.Assert._
+import org.junit._
+import org.junit.rules.TemporaryFolder
+
+import org.apache.kafka.streams.KeyValue
+import org.apache.kafka.streams._
+import org.apache.kafka.streams.scala.kstream._
+
+import org.apache.kafka.streams.integration.utils.{EmbeddedKafkaCluster, IntegrationTestUtils}
+import org.apache.kafka.clients.consumer.ConsumerConfig
+import org.apache.kafka.clients.producer.ProducerConfig
+
+import org.apache.kafka.common.serialization._
+import org.apache.kafka.common.utils.MockTime
+import org.apache.kafka.test.TestUtils
+
+import ImplicitConversions._
+import com.typesafe.scalalogging.LazyLogging
+
+/**
+ * Test suite that does a classic word count example.
+ * 
    
+ * The suite contains the test case using Scala APIs `testShouldCountWords` and the same test case using the
+ * Java APIs `testShouldCountWordsJava`. The idea is to demonstrate that both generate the same result.
+ * 
    
+ * Note: In the current project settings SAM type conversion is turned off as it's experimental in Scala 2.11.
+ * Hence the native Java API based version is more verbose.
+ */ 
+class WordCountTest extends JUnitSuite with WordCountTestData with LazyLogging {
+
+  private val privateCluster: EmbeddedKafkaCluster = new EmbeddedKafkaCluster(1)
+
+  @Rule def cluster: EmbeddedKafkaCluster = privateCluster
+
+  final val alignedTime = (System.currentTimeMillis() / 1000 + 1) * 1000
+  val mockTime: MockTime = cluster.time
+  mockTime.setCurrentTimeMs(alignedTime)
+
+
+  val tFolder: TemporaryFolder = new TemporaryFolder(TestUtils.tempDirectory())
+  @Rule def testFolder: TemporaryFolder = tFolder
+    
+
+  @Before
+  def startKafkaCluster(): Unit = {
+    cluster.createTopic(inputTopic)
+    cluster.createTopic(outputTopic)
+    cluster.createTopic(inputTopicJ)
+    cluster.createTopic(outputTopicJ)
+  }
+
+  @Test def testShouldCountWords(): Unit = {
+
+    import DefaultSerdes._
+
+    val streamsConfiguration = getStreamsConfiguration()
+
+    val streamBuilder = new StreamsBuilder
+    val textLines = streamBuilder.stream[String, String](inputTopic)
+
+    val pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS)
+
+    // generate word counts
+    val wordCounts: KTable[String, Long] =
+      textLines.flatMapValues(v => pattern.split(v.toLowerCase))
+        .groupBy((k, v) => v)
+        .count()
+
+    // write to output topic
+    wordCounts.toStream.to(outputTopic)
+
+    val streams: KafkaStreams = new KafkaStreams(streamBuilder.build(), streamsConfiguration)
+    streams.start()
+
+    // produce and consume synchronously
+    val actualWordCounts: java.util.List[KeyValue[String, Long]] = produceNConsume(inputTopic, outputTopic)
+
+    streams.close()
+
+    import collection.JavaConverters._
+    assertEquals(actualWordCounts.asScala.take(expectedWordCounts.size).sortBy(_.key), expectedWordCounts.sortBy(_.key))
+  }
+
+  @Test def testShouldCountWordsJava(): Unit = {
+
+    import org.apache.kafka.streams.{KafkaStreams => KafkaStreamsJ, StreamsBuilder => StreamsBuilderJ}
+    import org.apache.kafka.streams.kstream.{KTable => KTableJ, KStream => KStreamJ, KGroupedStream => KGroupedStreamJ, _}
+    import collection.JavaConverters._
+
+    val streamsConfiguration = getStreamsConfiguration()
+    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName())
+    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName())
+
+    val streamBuilder = new StreamsBuilderJ
+    val textLines: KStreamJ[String, String] = streamBuilder.stream[String, String](inputTopicJ)
+
+    val pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS)
+
+    val splits: KStreamJ[String, String] = textLines.flatMapValues {
+      new ValueMapper[String, java.lang.Iterable[String]] {
+        def apply(s: String): java.lang.Iterable[String] = pattern.split(s.toLowerCase).toIterable.asJava
+      }
+    }
+
+    val grouped: KGroupedStreamJ[String, String] = splits.groupBy {
+      new KeyValueMapper[String, String, String] {
+        def apply(k: String, v: String): String = v
+      }
+    }
+
+    val wordCounts: KTableJ[String, java.lang.Long] = grouped.count()
+
+    wordCounts.toStream.to(outputTopicJ, Produced.`with`(Serdes.String(), Serdes.Long()))
+
+    val streams: KafkaStreamsJ = new KafkaStreamsJ(streamBuilder.build(), streamsConfiguration)
+    streams.start()
+
+    val actualWordCounts: java.util.List[KeyValue[String, Long]] = produceNConsume(inputTopicJ, outputTopicJ)
+
+    streams.close()
+
+    assertEquals(actualWordCounts.asScala.take(expectedWordCounts.size).sortBy(_.key), expectedWordCounts.sortBy(_.key))
+  }
+
+  private def getStreamsConfiguration(): Properties = {
+    val streamsConfiguration: Properties = new Properties()
+
+    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "wordcount-test")
+    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, "10000")
+    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, testFolder.getRoot().getPath())
+    streamsConfiguration
+  }
+
+  private def getProducerConfig(): Properties = {
+    val p = new Properties()
+    p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    p.put(ProducerConfig.ACKS_CONFIG, "all")
+    p.put(ProducerConfig.RETRIES_CONFIG, "0")
+    p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
+    p.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
+    p
+  }
+
+  private def getConsumerConfig(): Properties = {
+    val p = new Properties()
+    p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers())
+    p.put(ConsumerConfig.GROUP_ID_CONFIG, "wordcount-scala-integration-test-standard-consumer")
+    p.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer])
+    p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[LongDeserializer])
+    p
+  }
+
+  private def produceNConsume(inputTopic: String, outputTopic: String): java.util.List[KeyValue[String, Long]] = {
+
+    val linesProducerConfig: Properties = getProducerConfig()
+
+    import collection.JavaConverters._
+    IntegrationTestUtils.produceValuesSynchronously(inputTopic, inputValues.asJava, linesProducerConfig, mockTime)
+
+    val consumerConfig = getConsumerConfig()
+
+    IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(consumerConfig, outputTopic, expectedWordCounts.size)
+  }
+}
+
+trait WordCountTestData {
+  val inputTopic = s"inputTopic"
+  val outputTopic = s"outputTopic"
+  val inputTopicJ = s"inputTopicJ"
+  val outputTopicJ = s"outputTopicJ"
+
+  val inputValues = List(
+    "Hello Kafka Streams",
+    "All streams lead to Kafka",
+    "Join Kafka Summit",
+    "И теперь пошли русские слова"
+  )
+
+  val expectedWordCounts: List[KeyValue[String, Long]] = List(
+    new KeyValue("hello", 1L),
+    new KeyValue("all", 1L),
+    new KeyValue("streams", 2L),
+    new KeyValue("lead", 1L),
+    new KeyValue("to", 1L),
+    new KeyValue("join", 1L),
+    new KeyValue("kafka", 3L),
+    new KeyValue("summit", 1L),
+    new KeyValue("и", 1L),
+    new KeyValue("теперь", 1L),
+    new KeyValue("пошли", 1L),
+    new KeyValue("русские", 1L),
+    new KeyValue("слова", 1L)
+  )
+}
+

From a05e33693b66ac38ccb21f2238c194ca59fcb6ec Mon Sep 17 00:00:00 2001
From: Jagadesh Adireddi 
Date: Tue, 24 Apr 2018 23:43:12 +0530
Subject: [PATCH 0287/1847] KAFKA-6677: Fixed StreamsConfig producer's
 max-in-flight allowed when EOS enabled. (#4868)

Reviewers: Matthias J Sax , Bill Bejeck 
---
 .../apache/kafka/streams/StreamsConfig.java   |  8 +++--
 .../kafka/streams/StreamsConfigTest.java      | 30 ++++++++-----------
 2 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index 65b1da6dedeef..e46d6d086ea24 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -27,6 +27,7 @@
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.Importance;
 import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.config.TopicConfig;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.serialization.Serde;
@@ -119,7 +120,7 @@
  * 
      
  *   
    • {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG "isolation.level"} (read_committed) - Consumers will always read committed data only
      • 
  *   
    • {@link ProducerConfig#ENABLE_IDEMPOTENCE_CONFIG "enable.idempotence"} (true) - Producer will always have idempotency enabled
      • 
- *   
    • {@link ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION "max.in.flight.requests.per.connection"} (1) - Producer will always have one in-flight request per connection
      • 
+ *   
    • {@link ProducerConfig#MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION "max.in.flight.requests.per.connection"} (5) - Producer will always have one in-flight request per connection
      • 
  * 
    
  *
  *
@@ -650,7 +651,6 @@ public class StreamsConfig extends AbstractConfig {
         final Map tempProducerDefaultOverrides = new HashMap<>(PRODUCER_DEFAULT_OVERRIDES);
         tempProducerDefaultOverrides.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE);
         tempProducerDefaultOverrides.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
-        tempProducerDefaultOverrides.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1);
 
         PRODUCER_EOS_OVERRIDES = Collections.unmodifiableMap(tempProducerDefaultOverrides);
     }
@@ -785,6 +785,10 @@ private void checkIfUnexpectedUserSpecifiedConsumerConfig(final Map producerConfigs = streamsConfig.getProducerConfigs("clientId");
-        assertThat((Integer) producerConfigs.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), equalTo(1));
-    }
-
-    @Test
-    public void shouldAllowSettingProducerMaxInFlightRequestPerConnectionsWhenEosDisabled() {
-        props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 2);
-        final StreamsConfig streamsConfig = new StreamsConfig(props);
-        final Map producerConfigs = streamsConfig.getProducerConfigs("clientId");
-        assertThat((Integer) producerConfigs.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), equalTo(2));
-    }
-
     @Test
     public void shouldSetDifferentDefaultsIfEosEnabled() {
         props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
@@ -446,7 +429,6 @@ public void shouldSetDifferentDefaultsIfEosEnabled() {
         assertThat((String) consumerConfigs.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG), equalTo(READ_COMMITTED.name().toLowerCase(Locale.ROOT)));
         assertTrue((Boolean) producerConfigs.get(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG));
         assertThat((Integer) producerConfigs.get(ProducerConfig.RETRIES_CONFIG), equalTo(Integer.MAX_VALUE));
-        assertThat((Integer) producerConfigs.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), equalTo(1));
         assertThat(streamsConfig.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG), equalTo(100L));
     }
 
@@ -563,6 +545,18 @@ public void shouldSpecifyCorrectValueSerdeClassOnError() {
         }
     }
 
+    @Test
+    public void shouldThrowExceptionIfMaxInflightRequestsGreatherThanFiveIfEosEnabled() {
+        props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 7);
+        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
+        final StreamsConfig streamsConfig = new StreamsConfig(props);
+        try {
+            streamsConfig.getProducerConfigs("clientId");
+            fail("Should throw ConfigException when Eos is enabled and maxInFlight requests exceeds 5");
+        } catch (final ConfigException e) {
+            assertEquals("max.in.flight.requests.per.connection can't exceed 5 when using the idempotent producer", e.getMessage());
+        }
+    }
 
 
     static class MisconfiguredSerde implements Serde {

From 3bc2575dfcfbf856b75c510d986f270f7f9411ca Mon Sep 17 00:00:00 2001
From: Anna Povzner 
Date: Tue, 24 Apr 2018 15:24:07 -0700
Subject: [PATCH 0288/1847] MINOR: Disabled flaky
 DynamicBrokerReconfigurationTest.testAddRemoveSslListener until fixed (#4924)

Reviewers: Colin Patrick McCabe , Jason Gustafson 
---
 .../kafka/server/DynamicBrokerReconfigurationTest.scala        | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index 79dec265f9062..ee0c05e25fcf8 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -54,7 +54,7 @@ import org.apache.kafka.common.record.TimestampType
 import org.apache.kafka.common.security.auth.SecurityProtocol
 import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
 import org.junit.Assert._
-import org.junit.{After, Before, Test}
+import org.junit.{After, Before, Test, Ignore}
 
 import scala.collection._
 import scala.collection.mutable.ArrayBuffer
@@ -652,6 +652,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet
   }
 
   @Test
+  @Ignore // Re-enable once we make it less flaky (KAFKA-6824)
   def testAddRemoveSslListener(): Unit = {
     verifyAddListener("SSL", SecurityProtocol.SSL, Seq.empty)
 

From 12a0f46895e730f6a3a4c5fc311c206f6d8219e5 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Tue, 24 Apr 2018 17:40:16 -0500
Subject: [PATCH 0289/1847] KAFKA-6376: Document skipped records metrics
 changes (#4922)

Reviewers: Bill Bejeck , Guozhang Wang 
---
 docs/ops.html                   |  7 ++++++-
 docs/streams/upgrade-guide.html | 31 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/docs/ops.html b/docs/ops.html
index 6ffe97653e6ea..450a268a2a1a2 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -1353,7 +1353,12 @@ 
    Streams API changes in 1.2.0
    
+    
    
+        We have removed the skippedDueToDeserializationError-rate and skippedDueToDeserializationError-total metrics.
+        Deserialization errors, and all other causes of record skipping, are now accounted for in the pre-existing metrics
+        skipped-records-rate and skipped-records-total. When a record is skipped, the event is
+        now logged at WARN level. If these warnings become burdensome, we recommend explicitly filtering out unprocessable
+        records instead of depending on record skipping semantics. For more details, see
+        KIP-274.
+        As of right now, the potential causes of skipped records are:
+    
    
+    
      
+        
    • null keys in table sources
      • 
+        
    • null keys in table-table inner/left/outer/right joins
      • 
+        
    • null keys or values in stream-table joins
      • 
+        
    • null keys or values in stream-stream joins
      • 
+        
    • null keys or values in aggregations on grouped streams
      • 
+        
    • null keys or values in reductions on grouped streams
      • 
+        
    • null keys in aggregations on windowed streams
      • 
+        
    • null keys in reductions on windowed streams
      • 
+        
    • null keys in aggregations on session-windowed streams
      • 
+        
    • 
+            Errors producing results, when the configured default.production.exception.handler decides to
+            CONTINUE (the default is to FAIL and throw an exception).
+        
      • 
+        
    • 
+            Errors deserializing records, when the configured default.deserialization.exception.handler
+            decides to CONTINUE (the default is to FAIL and throw an exception).
+            This was the case previously captured in the skippedDueToDeserializationError metrics.
+        
      • 
+        
    • Fetched records having a negative timestamp.
      • 
+    
    
+
     
    
         We have added support for methods in ReadOnlyWindowStore which allows for querying a single window's key-value pair.
         For users who have customized window store implementations on the above interface, they'd need to update their code to implement the newly added method as well.

From acd669e4248c605ac54c38862a8678f521f6f574 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Tue, 24 Apr 2018 21:49:44 -0700
Subject: [PATCH 0290/1847] KAFKA-6796; Fix surprising UNKNOWN_TOPIC error from
 requests to non-replicas (#4883)

Currently if the client sends a produce request or a fetch request to a broker which isn't a replica,
we return UNKNOWN_TOPIC_OR_PARTITION. This is a bit surprising to see when the topic actually
exists. It would be better to return NOT_LEADER to avoid confusion. Clients typically handle both errors by refreshing metadata and retrying, so changing this should not cause any change in behavior on the client. This case can be hit following a partition reassignment after the leader is moved and the local replica is deleted.

To validate the current behavior and the fix, I've added integration tests for the fetch and produce APIs.
---
 .../main/scala/kafka/server/KafkaApis.scala   |  32 ++---
 .../scala/kafka/server/ReplicaManager.scala   |  23 ++--
 .../unit/kafka/server/FetchRequestTest.scala  |  21 +++
 .../unit/kafka/server/KafkaApisTest.scala     | 127 +++++++++++++++---
 .../kafka/server/ProduceRequestTest.scala     |  23 ++++
 5 files changed, 176 insertions(+), 50 deletions(-)

diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index a0caa4a53c0a2..7bc9e3ee75039 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -287,7 +287,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       for ((topicPartition, partitionData) <- offsetCommitRequest.offsetData.asScala) {
         if (!authorize(request.session, Read, new Resource(Topic, topicPartition.topic)))
           unauthorizedTopicErrors += (topicPartition -> Errors.TOPIC_AUTHORIZATION_FAILED)
-        else if (!metadataCache.contains(topicPartition.topic))
+        else if (!metadataCache.contains(topicPartition))
           nonExistingTopicErrors += (topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION)
         else
           authorizedTopicRequestInfoBldr += (topicPartition -> partitionData)
@@ -401,7 +401,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     for ((topicPartition, memoryRecords) <- produceRequest.partitionRecordsOrFail.asScala) {
       if (!authorize(request.session, Write, new Resource(Topic, topicPartition.topic)))
         unauthorizedTopicResponses += topicPartition -> new PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED)
-      else if (!metadataCache.contains(topicPartition.topic))
+      else if (!metadataCache.contains(topicPartition))
         nonExistingTopicResponses += topicPartition -> new PartitionResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION)
       else
         authorizedRequestInfo += (topicPartition -> memoryRecords)
@@ -502,13 +502,13 @@ class KafkaApis(val requestChannel: RequestChannel,
     if (fetchRequest.isFromFollower()) {
       // The follower must have ClusterAction on ClusterResource in order to fetch partition data.
       if (authorize(request.session, ClusterAction, Resource.ClusterResource)) {
-        fetchContext.foreachPartition((part, data) => {
-          if (!metadataCache.contains(part.topic)) {
-            erroneous += part -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION,
+        fetchContext.foreachPartition((topicPartition, data) => {
+          if (!metadataCache.contains(topicPartition)) {
+            erroneous += topicPartition -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION,
               FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET,
               FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)
           } else {
-            interesting += (part -> data)
+            interesting += (topicPartition -> data)
           }
         })
       } else {
@@ -520,17 +520,17 @@ class KafkaApis(val requestChannel: RequestChannel,
       }
     } else {
       // Regular Kafka consumers need READ permission on each partition they are fetching.
-      fetchContext.foreachPartition((part, data) => {
-        if (!authorize(request.session, Read, new Resource(Topic, part.topic)))
-          erroneous += part -> new FetchResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED,
+      fetchContext.foreachPartition((topicPartition, data) => {
+        if (!authorize(request.session, Read, new Resource(Topic, topicPartition.topic)))
+          erroneous += topicPartition -> new FetchResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED,
             FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET,
             FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)
-        else if (!metadataCache.contains(part.topic))
-          erroneous += part -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION,
+        else if (!metadataCache.contains(topicPartition))
+          erroneous += topicPartition -> new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION,
             FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET,
             FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)
         else
-          interesting += (part -> data)
+          interesting += (topicPartition -> data)
       })
     }
 
@@ -1062,7 +1062,7 @@ class KafkaApis(val requestChannel: RequestChannel,
             // version 0 reads offsets from ZK
             val authorizedPartitionData = authorizedPartitions.map { topicPartition =>
               try {
-                if (!metadataCache.contains(topicPartition.topic))
+                if (!metadataCache.contains(topicPartition))
                   (topicPartition, OffsetFetchResponse.UNKNOWN_PARTITION)
                 else {
                   val payloadOpt = zkClient.getConsumerOffset(offsetFetchRequest.groupId, topicPartition)
@@ -1508,7 +1508,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       if (!authorize(request.session, Delete, new Resource(Topic, topicPartition.topic)))
         unauthorizedTopicResponses += topicPartition -> new DeleteRecordsResponse.PartitionResponse(
           DeleteRecordsResponse.INVALID_LOW_WATERMARK, Errors.TOPIC_AUTHORIZATION_FAILED)
-      else if (!metadataCache.contains(topicPartition.topic))
+      else if (!metadataCache.contains(topicPartition))
         nonExistingTopicResponses += topicPartition -> new DeleteRecordsResponse.PartitionResponse(
           DeleteRecordsResponse.INVALID_LOW_WATERMARK, Errors.UNKNOWN_TOPIC_OR_PARTITION)
       else
@@ -1720,7 +1720,7 @@ class KafkaApis(val requestChannel: RequestChannel,
         if (org.apache.kafka.common.internals.Topic.isInternal(topicPartition.topic) ||
             !authorize(request.session, Write, new Resource(Topic, topicPartition.topic)))
           unauthorizedTopicErrors += topicPartition -> Errors.TOPIC_AUTHORIZATION_FAILED
-        else if (!metadataCache.contains(topicPartition.topic))
+        else if (!metadataCache.contains(topicPartition))
           nonExistingTopicErrors += topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION
         else
           authorizedPartitions.add(topicPartition)
@@ -1806,7 +1806,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       for ((topicPartition, commitedOffset) <- txnOffsetCommitRequest.offsets.asScala) {
         if (!authorize(request.session, Read, new Resource(Topic, topicPartition.topic)))
           unauthorizedTopicErrors += topicPartition -> Errors.TOPIC_AUTHORIZATION_FAILED
-        else if (!metadataCache.contains(topicPartition.topic))
+        else if (!metadataCache.contains(topicPartition))
           nonExistingTopicErrors += topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION
         else
           authorizedTopicCommittedOffsets += (topicPartition -> commitedOffset)
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 0c2b0d805f8cc..da501174acdef 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -426,8 +426,16 @@ class ReplicaManager(val config: KafkaConfig,
   def getPartitionAndLeaderReplicaIfLocal(topicPartition: TopicPartition): (Partition, Replica) =  {
     val partitionOpt = getPartition(topicPartition)
     partitionOpt match {
+      case None if metadataCache.contains(topicPartition) =>
+        // The topic exists, but this broker is no longer a replica of it, so we return NOT_LEADER which
+        // forces clients to refresh metadata to find the new location. This can happen, for example,
+        // during a partition reassignment if a produce request from the client is sent to a broker after
+        // the local replica has been deleted.
+        throw new NotLeaderForPartitionException(s"Broker $localBrokerId is not a replica of $topicPartition")
+
       case None =>
-        throw new UnknownTopicOrPartitionException(s"Partition $topicPartition doesn't exist on $localBrokerId")
+        throw new UnknownTopicOrPartitionException(s"Partition $topicPartition doesn't exist")
+
       case Some(partition) =>
         if (partition eq ReplicaManager.OfflinePartition)
           throw new KafkaStorageException(s"Partition $topicPartition is in an offline log directory on broker $localBrokerId")
@@ -736,17 +744,8 @@ class ReplicaManager(val config: KafkaConfig,
           Some(new InvalidTopicException(s"Cannot append to internal topic ${topicPartition.topic}"))))
       } else {
         try {
-          val partitionOpt = getPartition(topicPartition)
-          val info = partitionOpt match {
-            case Some(partition) =>
-              if (partition eq ReplicaManager.OfflinePartition)
-                throw new KafkaStorageException(s"Partition $topicPartition is in an offline log directory on broker $localBrokerId")
-              partition.appendRecordsToLeader(records, isFromClient, requiredAcks)
-
-            case None => throw new UnknownTopicOrPartitionException("Partition %s doesn't exist on %d"
-              .format(topicPartition, localBrokerId))
-          }
-
+          val (partition, _) = getPartitionAndLeaderReplicaIfLocal(topicPartition)
+          val info = partition.appendRecordsToLeader(records, isFromClient, requiredAcks)
           val numAppendedMessages = info.numMessages
 
           // update stats for successfully appended bytes and messages as bytesInRate and messageInRate
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index f2b3552feaf5b..03137e10dea67 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -170,6 +170,27 @@ class FetchRequestTest extends BaseRequestTest {
     assertEquals(0, records(partitionData).map(_.sizeInBytes).sum)
   }
 
+  @Test
+  def testFetchRequestToNonReplica(): Unit = {
+    val topic = "topic"
+    val partition = 0
+    val topicPartition = new TopicPartition(topic, partition)
+
+    // Create a single-partition topic and find a broker which is not the leader
+    val partitionToLeader = TestUtils.createTopic(zkClient, topic, numPartitions = 1, 1, servers)
+    val leader = partitionToLeader(partition)
+    val nonReplicaOpt = servers.find(_.config.brokerId != leader)
+    assertTrue(nonReplicaOpt.isDefined)
+    val nonReplicaId =  nonReplicaOpt.get.config.brokerId
+
+    // Send the fetch request to the non-replica and verify the error code
+    val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024,
+      Seq(topicPartition))).build()
+    val fetchResponse = sendFetchRequest(nonReplicaId, fetchRequest)
+    val partitionData = fetchResponse.responseData.get(topicPartition)
+    assertEquals(Errors.NOT_LEADER_FOR_PARTITION, partitionData.error)
+  }
+
   /**
    * Tests that down-conversions dont leak memory. Large down conversions are triggered
    * in the server. The client closes its connection after reading partial data when the
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 382364f7a8341..96f74a0fffc96 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -20,6 +20,7 @@ package kafka.server
 import java.lang.{Long => JLong}
 import java.net.InetAddress
 import java.util
+import java.util.Collections
 
 import kafka.api.{ApiVersion, KAFKA_0_10_2_IV0}
 import kafka.cluster.Replica
@@ -40,6 +41,7 @@ import org.apache.kafka.common.network.ListenerName
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.RecordBatch
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
+import org.apache.kafka.common.requests.UpdateMetadataRequest.{Broker, EndPoint}
 import org.apache.kafka.common.requests.WriteTxnMarkersRequest.TxnMarkerEntry
 import org.apache.kafka.common.requests._
 import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
@@ -106,6 +108,84 @@ class KafkaApisTest {
     )
   }
 
+  @Test
+  def testOffsetCommitWithInvalidPartition(): Unit = {
+    val topic = "topic"
+    setupBasicMetadataCache(topic, numPartitions = 1)
+
+    def checkInvalidPartition(invalidPartitionId: Int): Unit = {
+      EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel)
+
+      val invalidTopicPartition = new TopicPartition(topic, invalidPartitionId)
+      val partitionOffsetCommitData = new OffsetCommitRequest.PartitionData(15L, "")
+      val (offsetCommitRequest, request) = buildRequest(new OffsetCommitRequest.Builder("groupId",
+        Map(invalidTopicPartition -> partitionOffsetCommitData).asJava))
+
+      val capturedResponse = expectThrottleCallbackAndInvoke()
+      EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel)
+      createKafkaApis().handleOffsetCommitRequest(request)
+
+      val response = readResponse(ApiKeys.OFFSET_COMMIT, offsetCommitRequest, capturedResponse)
+        .asInstanceOf[OffsetCommitResponse]
+      assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.responseData().get(invalidTopicPartition))
+    }
+
+    checkInvalidPartition(-1)
+    checkInvalidPartition(1) // topic has only one partition
+  }
+
+  @Test
+  def testTxnOffsetCommitWithInvalidPartition(): Unit = {
+    val topic = "topic"
+    setupBasicMetadataCache(topic, numPartitions = 1)
+
+    def checkInvalidPartition(invalidPartitionId: Int): Unit = {
+      EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel)
+
+      val invalidTopicPartition = new TopicPartition(topic, invalidPartitionId)
+      val partitionOffsetCommitData = new TxnOffsetCommitRequest.CommittedOffset(15L, "")
+      val (offsetCommitRequest, request) = buildRequest(new TxnOffsetCommitRequest.Builder("txnlId", "groupId",
+        15L, 0.toShort, Map(invalidTopicPartition -> partitionOffsetCommitData).asJava))
+
+      val capturedResponse = expectThrottleCallbackAndInvoke()
+      EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel)
+      createKafkaApis().handleTxnOffsetCommitRequest(request)
+
+      val response = readResponse(ApiKeys.TXN_OFFSET_COMMIT, offsetCommitRequest, capturedResponse)
+        .asInstanceOf[TxnOffsetCommitResponse]
+      assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.errors().get(invalidTopicPartition))
+    }
+
+    checkInvalidPartition(-1)
+    checkInvalidPartition(1) // topic has only one partition
+  }
+
+  @Test
+  def testAddPartitionsToTxnWithInvalidPartition(): Unit = {
+    val topic = "topic"
+    setupBasicMetadataCache(topic, numPartitions = 1)
+
+    def checkInvalidPartition(invalidPartitionId: Int): Unit = {
+      EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel)
+
+      val invalidTopicPartition = new TopicPartition(topic, invalidPartitionId)
+
+      val (addPartitionsToTxnRequest, request) = buildRequest(new AddPartitionsToTxnRequest.Builder(
+        "txnlId", 15L, 0.toShort, List(invalidTopicPartition).asJava))
+
+      val capturedResponse = expectThrottleCallbackAndInvoke()
+      EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel)
+      createKafkaApis().handleAddPartitionToTxnRequest(request)
+
+      val response = readResponse(ApiKeys.ADD_PARTITIONS_TO_TXN, addPartitionsToTxnRequest, capturedResponse)
+        .asInstanceOf[AddPartitionsToTxnResponse]
+      assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.errors().get(invalidTopicPartition))
+    }
+
+    checkInvalidPartition(-1)
+    checkInvalidPartition(1) // topic has only one partition
+  }
+
   @Test(expected = classOf[UnsupportedVersionException])
   def shouldThrowUnsupportedVersionExceptionOnHandleAddOffsetToTxnRequestWhenInterBrokerProtocolNotSupported(): Unit = {
     createKafkaApis(KAFKA_0_10_2_IV0).handleAddOffsetsToTxnRequest(null)
@@ -284,8 +364,6 @@ class KafkaApisTest {
     val timestamp: JLong = time.milliseconds()
     val limitOffset = 15L
 
-    val capturedResponse = EasyMock.newCapture[RequestChannel.Response]()
-    val capturedThrottleCallback = EasyMock.newCapture[Int => Unit]()
     val replica = EasyMock.mock(classOf[Replica])
     val log = EasyMock.mock(classOf[Log])
     EasyMock.expect(replicaManager.getLeaderReplicaIfLocal(tp)).andReturn(replica)
@@ -295,8 +373,7 @@ class KafkaApisTest {
       EasyMock.expect(replica.lastStableOffset).andReturn(LogOffsetMetadata(messageOffset = limitOffset))
     EasyMock.expect(replicaManager.getLog(tp)).andReturn(Some(log))
     EasyMock.expect(log.fetchOffsetsByTimestamp(timestamp)).andReturn(Some(TimestampOffset(timestamp = timestamp, offset = limitOffset)))
-    expectThrottleCallbackAndInvoke(capturedThrottleCallback)
-    EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse)))
+    val capturedResponse = expectThrottleCallbackAndInvoke()
     EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, replica, log)
 
     val builder = ListOffsetRequest.Builder.forConsumer(true, isolationLevel)
@@ -327,8 +404,6 @@ class KafkaApisTest {
     val tp = new TopicPartition("foo", 0)
     val limitOffset = 15L
 
-    val capturedResponse = EasyMock.newCapture[RequestChannel.Response]()
-    val capturedThrottleCallback = EasyMock.newCapture[Int => Unit]()
     val replica = EasyMock.mock(classOf[Replica])
     val log = EasyMock.mock(classOf[Log])
     EasyMock.expect(replicaManager.getLeaderReplicaIfLocal(tp)).andReturn(replica)
@@ -339,8 +414,7 @@ class KafkaApisTest {
     EasyMock.expect(replicaManager.getLog(tp)).andReturn(Some(log))
     EasyMock.expect(log.fetchOffsetsByTimestamp(ListOffsetRequest.EARLIEST_TIMESTAMP))
       .andReturn(Some(TimestampOffset(timestamp = ListOffsetResponse.UNKNOWN_TIMESTAMP, offset = limitOffset)))
-    expectThrottleCallbackAndInvoke(capturedThrottleCallback)
-    EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse)))
+    val capturedResponse = expectThrottleCallbackAndInvoke()
     EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, replica, log)
 
     val builder = ListOffsetRequest.Builder.forConsumer(true, isolationLevel)
@@ -393,14 +467,12 @@ class KafkaApisTest {
    * Return pair of listener names in the metadataCache: PLAINTEXT and LISTENER2 respectively.
    */
   private def updateMetadataCacheWithInconsistentListeners(): (ListenerName, ListenerName) = {
-    import UpdateMetadataRequest.{Broker => UBroker}
-    import UpdateMetadataRequest.{EndPoint => UEndPoint}
     val plaintextListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
     val anotherListener = new ListenerName("LISTENER2")
     val brokers = Set(
-      new UBroker(0, Seq(new UEndPoint("broker0", 9092, SecurityProtocol.PLAINTEXT, plaintextListener),
-        new UEndPoint("broker0", 9093, SecurityProtocol.PLAINTEXT, anotherListener)).asJava, "rack"),
-      new UBroker(1, Seq(new UEndPoint("broker1", 9092, SecurityProtocol.PLAINTEXT, plaintextListener)).asJava,
+      new Broker(0, Seq(new EndPoint("broker0", 9092, SecurityProtocol.PLAINTEXT, plaintextListener),
+        new EndPoint("broker0", 9093, SecurityProtocol.PLAINTEXT, anotherListener)).asJava, "rack"),
+      new Broker(1, Seq(new EndPoint("broker1", 9092, SecurityProtocol.PLAINTEXT, plaintextListener)).asJava,
         "rack")
     )
     val updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, 0,
@@ -410,10 +482,7 @@ class KafkaApisTest {
   }
 
   private def sendMetadataRequestWithInconsistentListeners(requestListener: ListenerName): MetadataResponse = {
-    val capturedResponse = EasyMock.newCapture[RequestChannel.Response]()
-    val capturedThrottleCallback = EasyMock.newCapture[Int => Unit]()
-    expectThrottleCallbackAndInvoke(capturedThrottleCallback)
-    EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse)))
+    val capturedResponse = expectThrottleCallbackAndInvoke()
     EasyMock.replay(clientRequestQuotaManager, requestChannel)
 
     val (metadataRequest, requestChannelRequest) = buildRequest(MetadataRequest.Builder.allTopics, requestListener)
@@ -426,8 +495,6 @@ class KafkaApisTest {
     val tp = new TopicPartition("foo", 0)
     val latestOffset = 15L
 
-    val capturedResponse = EasyMock.newCapture[RequestChannel.Response]()
-    val capturedThrottleCallback = EasyMock.newCapture[Int => Unit]()
     val replica = EasyMock.mock(classOf[Replica])
     val log = EasyMock.mock(classOf[Log])
     EasyMock.expect(replicaManager.getLeaderReplicaIfLocal(tp)).andReturn(replica)
@@ -435,8 +502,8 @@ class KafkaApisTest {
       EasyMock.expect(replica.highWatermark).andReturn(LogOffsetMetadata(messageOffset = latestOffset))
     else
       EasyMock.expect(replica.lastStableOffset).andReturn(LogOffsetMetadata(messageOffset = latestOffset))
-    expectThrottleCallbackAndInvoke(capturedThrottleCallback)
-    EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse)))
+
+    val capturedResponse = expectThrottleCallbackAndInvoke()
     EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, replica, log)
 
     val builder = ListOffsetRequest.Builder.forConsumer(true, isolationLevel)
@@ -484,7 +551,8 @@ class KafkaApisTest {
     AbstractResponse.parseResponse(api, struct)
   }
 
-  private def expectThrottleCallbackAndInvoke(capturedThrottleCallback: Capture[Int => Unit]): Unit = {
+  private def expectThrottleCallbackAndInvoke(): Capture[RequestChannel.Response] = {
+    val capturedThrottleCallback = EasyMock.newCapture[Int => Unit]()
     EasyMock.expect(clientRequestQuotaManager.maybeRecordAndThrottle(
       EasyMock.anyObject[RequestChannel.Request](),
       EasyMock.capture(capturedThrottleCallback)))
@@ -494,6 +562,21 @@ class KafkaApisTest {
           callback(0)
         }
       })
+
+    val capturedResponse = EasyMock.newCapture[RequestChannel.Response]()
+    EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse)))
+    capturedResponse
+  }
+
+  private def setupBasicMetadataCache(topic: String, numPartitions: Int = 1): Unit = {
+    val replicas = List(0.asInstanceOf[Integer]).asJava
+    val partitionState = new UpdateMetadataRequest.PartitionState(1, 0, 1, replicas, 0, replicas, Collections.emptyList())
+    val plaintextListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
+    val broker = new Broker(0, Seq(new EndPoint("broker0", 9092, SecurityProtocol.PLAINTEXT, plaintextListener)).asJava, "rack")
+    val partitions = (0 until numPartitions).map(new TopicPartition(topic, _) -> partitionState).toMap
+    val updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, 0,
+      0, partitions.asJava, Set(broker).asJava).build()
+    metadataCache.updateCache(correlationId = 0, updateMetadataRequest)
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
index 60244996dc889..4e66494374f71 100644
--- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
@@ -59,6 +59,29 @@ class ProduceRequestTest extends BaseRequestTest {
       new SimpleRecord(System.currentTimeMillis(), "key2".getBytes, "value2".getBytes)), 1)
   }
 
+  @Test
+  def testProduceToNonReplica() {
+    val topic = "topic"
+    val partition = 0
+
+    // Create a single-partition topic and find a broker which is not the leader
+    val partitionToLeader = TestUtils.createTopic(zkClient, topic, numPartitions = 1, 1, servers)
+    val leader = partitionToLeader(partition)
+    val nonReplicaOpt = servers.find(_.config.brokerId != leader)
+    assertTrue(nonReplicaOpt.isDefined)
+    val nonReplicaId =  nonReplicaOpt.get.config.brokerId
+
+    // Send the produce request to the non-replica
+    val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("key".getBytes, "value".getBytes))
+    val topicPartition = new TopicPartition("topic", partition)
+    val partitionRecords = Map(topicPartition -> records)
+    val produceRequest = ProduceRequest.Builder.forCurrentMagic(-1, 3000, partitionRecords.asJava).build()
+
+    val produceResponse = sendProduceRequest(nonReplicaId, produceRequest)
+    assertEquals(1, produceResponse.responses.size)
+    assertEquals(Errors.NOT_LEADER_FOR_PARTITION, produceResponse.responses.asScala.head._2.error)
+  }
+
   /* returns a pair of partition id and leader id */
   private def createTopicAndFindPartitionWithLeader(topic: String): (Int, Int) = {
     val partitionToLeader = TestUtils.createTopic(zkClient, topic, 3, 2, servers)

From c853ef75a11663cbee3160e64e45f46f5a8ac78c Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Wed, 25 Apr 2018 03:19:31 -0700
Subject: [PATCH 0291/1847] MINOR: Bump version to 2.0.0-SNAPSHOT (#4804)

---
 core/src/main/scala/kafka/api/ApiVersion.scala | 10 +++++++++-
 gradle.properties                              |  2 +-
 kafka-merge-pr.py                              |  2 +-
 tests/kafkatest/__init__.py                    |  2 +-
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala
index 9270a7a3b0120..62b91a0eb8b5a 100644
--- a/core/src/main/scala/kafka/api/ApiVersion.scala
+++ b/core/src/main/scala/kafka/api/ApiVersion.scala
@@ -76,7 +76,9 @@ object ApiVersion {
     // Introduced DeleteGroupsRequest V0 via KIP-229, plus KIP-227 incremental fetch requests,
     // and KafkaStorageException for fetch requests.
     "1.1-IV0" -> KAFKA_1_1_IV0,
-    "1.1" -> KAFKA_1_1_IV0
+    "1.1" -> KAFKA_1_1_IV0,
+    "2.0-IV0" -> KAFKA_2_0_IV0,
+    "2.0" -> KAFKA_2_0_IV0
   )
 
   private val versionPattern = "\\.".r
@@ -205,3 +207,9 @@ case object KAFKA_1_1_IV0 extends ApiVersion {
   val messageFormatVersion = RecordFormat.V2
   val id: Int = 14
 }
+
+case object KAFKA_2_0_IV0 extends ApiVersion {
+  val version: String = "2.0-IV0"
+  val messageFormatVersion = RecordFormat.V2
+  val id: Int = 15
+}
diff --git a/gradle.properties b/gradle.properties
index 325a1d0056e63..ab94b7be083d7 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -16,7 +16,7 @@
 group=org.apache.kafka
 # NOTE: When you change this version number, you should also make sure to update
 # the version numbers in tests/kafkatest/__init__.py and kafka-merge-pr.py.
-version=1.2.0-SNAPSHOT
+version=2.0.0-SNAPSHOT
 scalaVersion=2.11.12
 task=build
 org.gradle.jvmargs=-XX:MaxPermSize=512m -Xmx1024m -Xss2m
diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
index 02cf6e02a3ce7..b2c468835b2af 100755
--- a/kafka-merge-pr.py
+++ b/kafka-merge-pr.py
@@ -70,7 +70,7 @@
 
 DEV_BRANCH_NAME = "trunk"
 
-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "1.2.0")
+DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "2.0.0")
 
 def get_json(url):
     try:
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
index 935f20d236fb3..8f7aadd12e5f6 100644
--- a/tests/kafkatest/__init__.py
+++ b/tests/kafkatest/__init__.py
@@ -22,4 +22,4 @@
 # Instead, in development branches, the version should have a suffix of the form ".devN"
 #
 # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
-__version__ = '1.2.0.dev0'
+__version__ = '2.0.0.dev0'

From 34a1f7099b65f43037602eb13cbffd7df276e290 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= 
Date: Wed, 25 Apr 2018 08:36:14 -0700
Subject: [PATCH 0292/1847] KAFKA-6826 avoid range scans when forwarding values
 during aggregation (#4927)

Reviewers: Matthias J Sax , Bill Bejeck , John Roesler , Guozhang Wang 
---
 .../streams/state/internals/CachingWindowStore.java    | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
index 9ef41ce20c7dc..58111a670d648 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
@@ -219,13 +219,11 @@ public KeyValueIterator, byte[]> fetch(final Bytes from, final B
     }
     
     private V fetchPrevious(final Bytes key, final long timestamp) {
-        try (final WindowStoreIterator iter = underlying.fetch(key, timestamp, timestamp)) {
-            if (!iter.hasNext()) {
-                return null;
-            } else {
-                return serdes.valueFrom(iter.next().value);
-            }
+        final byte[] value = underlying.fetch(key, timestamp);
+        if (value != null) {
+            return serdes.valueFrom(value);
         }
+        return null;
     }
     
     @Override

From cbb5b51475368613c7972297ea6055a4f75285e1 Mon Sep 17 00:00:00 2001
From: Anna Povzner 
Date: Wed, 25 Apr 2018 14:24:29 -0700
Subject: [PATCH 0293/1847] KAFKA-6795; Added unit tests for
 ReplicaAlterLogDirsThread

Added unit tests for ReplicaAlterLogDirsThread. Mostly focused on unit tests for truncating logic.

Fixed  ReplicaAlterLogDirsThread.buildLeaderEpochRequest() to use future replica's latest epoch (not the latest epoch of replica it is fetching from). This follows the logic that offset for leader epoch request should be based on leader epoch of the follower (in this case it's the future local replica).

Also fixed PartitionFetchState constructor that takes offset and delay. The code ignored the delay parameter and used 0 for the delay. This constructor is used only by another constructor which passes delay = 0, which luckily works.

Author: Anna Povzner 

Reviewers: Dong Lin 

Closes #4918 from apovzner/kafka-6795
---
 .../kafka/server/AbstractFetcherThread.scala  |   2 +-
 .../server/ReplicaAlterLogDirsThread.scala    |  15 +-
 .../ReplicaAlterLogDirsThreadTest.scala       | 526 ++++++++++++++++++
 3 files changed, 539 insertions(+), 4 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala

diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index 8d787c96da6e3..f919ddf017c92 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -434,7 +434,7 @@ case class PartitionFetchState(fetchOffset: Long, delay: DelayedItem, truncating
 
   def this(offset: Long, truncatingLog: Boolean) = this(offset, new DelayedItem(0), truncatingLog)
 
-  def this(offset: Long, delay: DelayedItem) = this(offset, new DelayedItem(0), false)
+  def this(offset: Long, delay: DelayedItem) = this(offset, delay, false)
 
   def this(fetchOffset: Long) = this(fetchOffset, new DelayedItem(0))
 
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 48c83d4383580..0faf5dc383804 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -58,8 +58,6 @@ class ReplicaAlterLogDirsThread(name: String,
   private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes
   private val fetchSize = brokerConfig.replicaFetchMaxBytes
 
-  private def epochCacheOpt(tp: TopicPartition): Option[LeaderEpochCache] =  replicaMgr.getReplica(tp).map(_.epochs.get)
-
   def fetch(fetchRequest: FetchRequest): Seq[(TopicPartition, PartitionData)] = {
     var partitionData: Seq[(TopicPartition, FetchResponse.PartitionData)] = null
     val request = fetchRequest.underlying.build()
@@ -141,7 +139,13 @@ class ReplicaAlterLogDirsThread(name: String,
       delayPartitions(partitions, brokerConfig.replicaFetchBackoffMs.toLong)
   }
 
+  /**
+   * Builds offset for leader epoch requests for partitions that are in the truncating phase based
+   * on latest epochs of the future replicas (the one that is fetching)
+   */
   def buildLeaderEpochRequest(allPartitions: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[Map[TopicPartition, Int]] = {
+    def epochCacheOpt(tp: TopicPartition): Option[LeaderEpochCache] = replicaMgr.getReplica(tp, Request.FutureLocalReplicaId).map(_.epochs.get)
+
     val partitionEpochOpts = allPartitions
       .filter { case (_, state) => state.isTruncatingLog }
       .map { case (tp, _) => tp -> epochCacheOpt(tp) }.toMap
@@ -152,6 +156,11 @@ class ReplicaAlterLogDirsThread(name: String,
     ResultWithPartitions(result, partitionsWithoutEpoch.keys.toSet)
   }
 
+  /**
+   * Fetches offset for leader epoch from local replica for each given topic partitions
+   * @param partitions map of topic partition -> leader epoch of the future replica
+   * @return map of topic partition -> end offset for a requested leader epoch
+   */
   def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
     partitions.map { case (tp, epoch) =>
       try {
@@ -263,4 +272,4 @@ object ReplicaAlterLogDirsThread {
 
     override def toString = underlying.toString
   }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
new file mode 100644
index 0000000000000..a0f1dae8c76e0
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
@@ -0,0 +1,526 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+package kafka.server
+
+
+import kafka.api.Request
+import kafka.cluster.{BrokerEndPoint, Replica, Partition}
+import kafka.log.LogManager
+import kafka.server.AbstractFetcherThread.ResultWithPartitions
+import kafka.server.FetchPartitionData
+import kafka.server.epoch.LeaderEpochCache
+import org.apache.kafka.common.errors.{ReplicaNotAvailableException, KafkaStorageException}
+import kafka.utils.{DelayedItem, TestUtils}
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchResponse, FetchMetadata => JFetchMetadata}
+import org.apache.kafka.common.requests.FetchResponse.PartitionData
+import org.apache.kafka.common.requests.EpochEndOffset.{UNDEFINED_EPOCH_OFFSET, UNDEFINED_EPOCH}
+import org.apache.kafka.common.utils.SystemTime
+import org.easymock.EasyMock._
+import org.easymock.{Capture, CaptureType, EasyMock, IAnswer}
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.JavaConverters._
+import scala.collection.Seq
+import scala.collection.{Map, mutable}
+
+class ReplicaAlterLogDirsThreadTest {
+
+  private val t1p0 = new TopicPartition("topic1", 0)
+  private val t1p1 = new TopicPartition("topic1", 1)
+
+  @Test
+  def issuesEpochRequestFromLocalReplica(): Unit = {
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+
+    //Setup all dependencies
+    val leaderEpochs = createNiceMock(classOf[LeaderEpochCache])
+    val replica = createNiceMock(classOf[Replica])
+    val futureReplica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+
+    val leaderEpoch = 2
+    val leo = 13
+
+    //Stubs
+    expect(replica.epochs).andReturn(Some(leaderEpochs)).anyTimes()
+    expect(leaderEpochs.endOffsetFor(leaderEpoch)).andReturn(leo).anyTimes()
+    stub(replica, replica, futureReplica, partition, replicaManager)
+
+    replay(leaderEpochs, replicaManager, replica)
+
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = null,
+      brokerTopicStats = null)
+
+    val result = thread.fetchEpochsFromLeader(Map(t1p0 -> leaderEpoch, t1p1 -> leaderEpoch))
+
+    val expected = Map(
+      t1p0 -> new EpochEndOffset(Errors.NONE, leo),
+      t1p1 -> new EpochEndOffset(Errors.NONE, leo)
+    )
+
+    assertEquals("results from leader epoch request should have offset from local replica",
+                 expected, result)
+  }
+
+  @Test
+  def fetchEpochsFromLeaderShouldHandleExceptionFromGetLocalReplica(): Unit = {
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+
+    //Setup all dependencies
+    val leaderEpochs = createNiceMock(classOf[LeaderEpochCache])
+    val replica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+
+    val leaderEpoch = 2
+    val leo = 13
+
+    //Stubs
+    expect(replica.epochs).andReturn(Some(leaderEpochs)).anyTimes()
+    expect(leaderEpochs.endOffsetFor(leaderEpoch)).andReturn(leo).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p0)).andReturn(replica).anyTimes()
+    expect(replicaManager.getPartition(t1p0)).andReturn(Some(partition)).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p1)).andThrow(new KafkaStorageException).once()
+    expect(replicaManager.getPartition(t1p1)).andReturn(Some(partition)).anyTimes()
+
+    replay(leaderEpochs, replicaManager, replica)
+
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = null,
+      brokerTopicStats = null)
+
+    val result = thread.fetchEpochsFromLeader(Map(t1p0 -> leaderEpoch, t1p1 -> leaderEpoch))
+
+    val expected = Map(
+      t1p0 -> new EpochEndOffset(Errors.NONE, leo),
+      t1p1 -> new EpochEndOffset(Errors.KAFKA_STORAGE_ERROR, UNDEFINED_EPOCH_OFFSET)
+    )
+
+    assertEquals(expected, result)
+  }
+
+  @Test
+  def shouldTruncateToReplicaOffset(): Unit = {
+
+    //Create a capture to track what partitions/offsets are truncated
+    val truncateToCapture: Capture[Long] = newCapture(CaptureType.ALL)
+
+    // Setup all the dependencies
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+    val quotaManager = createNiceMock(classOf[ReplicationQuotaManager])
+    val leaderEpochsT1p0 = createMock(classOf[LeaderEpochCache])
+    val leaderEpochsT1p1 = createMock(classOf[LeaderEpochCache])
+    val futureReplicaLeaderEpochs = createMock(classOf[LeaderEpochCache])
+    val logManager = createMock(classOf[LogManager])
+    val replicaT1p0 = createNiceMock(classOf[Replica])
+    val replicaT1p1 = createNiceMock(classOf[Replica])
+    // one future replica mock because our mocking methods return same values for both future replicas
+    val futureReplica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+    val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
+
+    val leaderEpoch = 2
+    val futureReplicaLEO = 191
+    val replicaT1p0LEO = 190
+    val replicaT1p1LEO = 192
+
+    //Stubs
+    expect(partition.truncateTo(capture(truncateToCapture), anyBoolean())).anyTimes()
+    expect(replicaT1p0.epochs).andReturn(Some(leaderEpochsT1p0)).anyTimes()
+    expect(replicaT1p1.epochs).andReturn(Some(leaderEpochsT1p1)).anyTimes()
+    expect(futureReplica.epochs).andReturn(Some(futureReplicaLeaderEpochs)).anyTimes()
+    expect(futureReplica.logEndOffset).andReturn(new LogOffsetMetadata(futureReplicaLEO)).anyTimes()
+    expect(futureReplicaLeaderEpochs.latestEpoch).andReturn(leaderEpoch).anyTimes()
+    expect(leaderEpochsT1p0.endOffsetFor(leaderEpoch)).andReturn(replicaT1p0LEO).anyTimes()
+    expect(leaderEpochsT1p1.endOffsetFor(leaderEpoch)).andReturn(replicaT1p1LEO).anyTimes()
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    stubWithFetchMessages(replicaT1p0, replicaT1p1, futureReplica, partition, replicaManager, responseCallback)
+
+    replay(leaderEpochsT1p0, leaderEpochsT1p1, futureReplicaLeaderEpochs, replicaManager,
+           logManager, quotaManager, replicaT1p0, replicaT1p1, futureReplica, partition)
+
+    //Create the thread
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = null)
+    thread.addPartitions(Map(t1p0 -> 0, t1p1 -> 0))
+
+    //Run it
+    thread.doWork()
+
+    //We should have truncated to the offsets in the response
+    assertTrue(truncateToCapture.getValues.asScala.contains(replicaT1p0LEO))
+    assertTrue(truncateToCapture.getValues.asScala.contains(futureReplicaLEO))
+  }
+
+  @Test
+  def shouldTruncateToInitialFetchOffsetIfReplicaReturnsUndefinedOffset(): Unit = {
+
+    //Create a capture to track what partitions/offsets are truncated
+    val truncated: Capture[Long] = newCapture(CaptureType.ALL)
+
+    // Setup all the dependencies
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+    val quotaManager = createNiceMock(classOf[ReplicationQuotaManager])
+    val logManager = createMock(classOf[LogManager])
+    val replica = createNiceMock(classOf[Replica])
+    val futureReplica = createNiceMock(classOf[Replica])
+    val leaderEpochs = createNiceMock(classOf[LeaderEpochCache])
+    val futureReplicaLeaderEpochs = createMock(classOf[LeaderEpochCache])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+    val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
+
+    val initialFetchOffset = 100
+    val futureReplicaLEO = 111
+
+    //Stubs
+    expect(partition.truncateTo(capture(truncated), anyBoolean())).anyTimes()
+    expect(futureReplica.logEndOffset).andReturn(new LogOffsetMetadata(futureReplicaLEO)).anyTimes()
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    expect(replica.epochs).andReturn(Some(leaderEpochs)).anyTimes()
+    expect(futureReplica.epochs).andReturn(Some(futureReplicaLeaderEpochs)).anyTimes()
+
+    // pretend this is a completely new future replica, with no leader epochs recorded
+    expect(futureReplicaLeaderEpochs.latestEpoch).andReturn(UNDEFINED_EPOCH).anyTimes()
+
+    // since UNDEFINED_EPOCH is -1 wich will be lower than any valid leader epoch, the method
+    // will return UNDEFINED_EPOCH_OFFSET if requested epoch is lower than the first epoch cached
+    expect(leaderEpochs.endOffsetFor(UNDEFINED_EPOCH)).andReturn(UNDEFINED_EPOCH_OFFSET).anyTimes()
+    stubWithFetchMessages(replica, replica, futureReplica, partition, replicaManager, responseCallback)
+    replay(replicaManager, logManager, quotaManager, leaderEpochs, futureReplicaLeaderEpochs,
+           replica, futureReplica, partition)
+
+    //Create the thread
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = null)
+    thread.addPartitions(Map(t1p0 -> initialFetchOffset))
+
+    //Run it
+    thread.doWork()
+
+    //We should have truncated to initial fetch offset
+    assertEquals("Expected future replica to truncate to initial fetch offset if replica returns UNDEFINED_EPOCH_OFFSET",
+                 initialFetchOffset, truncated.getValue)
+  }
+
+  @Test
+  def shouldPollIndefinitelyIfReplicaNotAvailable(): Unit = {
+
+    //Create a capture to track what partitions/offsets are truncated
+    val truncated: Capture[Long] = newCapture(CaptureType.ALL)
+
+    // Setup all the dependencies
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+    val quotaManager = createNiceMock(classOf[kafka.server.ReplicationQuotaManager])
+    val leaderEpochs = createNiceMock(classOf[LeaderEpochCache])
+    val futureReplicaLeaderEpochs = createMock(classOf[LeaderEpochCache])
+    val logManager = createMock(classOf[kafka.log.LogManager])
+    val replica = createNiceMock(classOf[Replica])
+    val futureReplica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[kafka.server.ReplicaManager])
+    val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
+
+    val futureReplicaLeaderEpoch = 1
+    val futureReplicaLEO = 290
+    val replicaLEO = 300
+
+    //Stubs
+    expect(partition.truncateTo(capture(truncated), anyBoolean())).anyTimes()
+    expect(replica.epochs).andReturn(Some(leaderEpochs)).anyTimes()
+    expect(futureReplica.epochs).andReturn(Some(futureReplicaLeaderEpochs)).anyTimes()
+
+    expect(futureReplicaLeaderEpochs.latestEpoch).andReturn(futureReplicaLeaderEpoch).anyTimes()
+    expect(leaderEpochs.endOffsetFor(futureReplicaLeaderEpoch)).andReturn(replicaLEO).anyTimes()
+
+    expect(futureReplica.logEndOffset).andReturn(new LogOffsetMetadata(futureReplicaLEO)).anyTimes()
+    expect(replicaManager.getReplica(t1p0)).andReturn(Some(replica)).anyTimes()
+    expect(replicaManager.getReplica(t1p0, Request.FutureLocalReplicaId)).andReturn(Some(futureReplica)).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p0, Request.FutureLocalReplicaId)).andReturn(futureReplica).anyTimes()
+    // this will cause fetchEpochsFromLeader return an error with undefined offset
+    expect(replicaManager.getReplicaOrException(t1p0)).andThrow(new ReplicaNotAvailableException("")).times(3)
+    expect(replicaManager.getReplicaOrException(t1p0)).andReturn(replica).once()
+    expect(replicaManager.getPartition(t1p0)).andReturn(Some(partition)).anyTimes()
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    expect(replicaManager.fetchMessages(
+      EasyMock.anyLong(),
+      EasyMock.anyInt(),
+      EasyMock.anyInt(),
+      EasyMock.anyInt(),
+      EasyMock.anyObject(),
+      EasyMock.anyObject(),
+      EasyMock.anyObject(),
+      EasyMock.capture(responseCallback),
+      EasyMock.anyObject()))
+      .andAnswer(new IAnswer[Unit] {
+        override def answer(): Unit = {
+          responseCallback.getValue.apply(Seq.empty[(TopicPartition, FetchPartitionData)])
+        }
+      }).anyTimes()
+
+    replay(leaderEpochs, futureReplicaLeaderEpochs, replicaManager, logManager, quotaManager,
+           replica, futureReplica, partition)
+
+    //Create the thread
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = null)
+    thread.addPartitions(Map(t1p0 -> 0))
+
+    // Run thread 3 times (exactly number of times we mock exception for getReplicaOrException)
+    (0 to 2).foreach { _ =>
+      thread.doWork()
+                     }
+
+    // Nothing happened since the replica was not available
+    assertEquals(0, truncated.getValues.size())
+
+    // Next time we loop, getReplicaOrException will return replica
+    thread.doWork()
+
+    // Now the final call should have actually done a truncation (to offset futureReplicaLEO)
+    assertEquals(futureReplicaLEO, truncated.getValue)
+  }
+
+  @Test
+  def shouldFetchLeaderEpochOnFirstFetchOnly(): Unit = {
+
+    //Setup all dependencies
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+    val quotaManager = createNiceMock(classOf[ReplicationQuotaManager])
+    val leaderEpochs = createNiceMock(classOf[LeaderEpochCache])
+    val futureReplicaLeaderEpochs = createMock(classOf[LeaderEpochCache])
+    val logManager = createMock(classOf[LogManager])
+    val replica = createNiceMock(classOf[Replica])
+    val futureReplica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+    val responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]  = EasyMock.newCapture()
+
+    val leaderEpoch = 5
+    val futureReplicaLEO = 190
+    val replicaLEO = 213
+
+    //Stubs
+    expect(partition.truncateTo(futureReplicaLEO, true)).once()
+    expect(replica.epochs).andReturn(Some(leaderEpochs)).anyTimes()
+    expect(futureReplica.epochs).andReturn(Some(futureReplicaLeaderEpochs)).anyTimes()
+
+    expect(futureReplica.logEndOffset).andReturn(new LogOffsetMetadata(futureReplicaLEO)).anyTimes()
+    expect(futureReplicaLeaderEpochs.latestEpoch).andReturn(leaderEpoch)
+    expect(leaderEpochs.endOffsetFor(leaderEpoch)).andReturn(replicaLEO)
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    stubWithFetchMessages(replica, replica, futureReplica, partition, replicaManager, responseCallback)
+
+    replay(leaderEpochs, futureReplicaLeaderEpochs, replicaManager, logManager, quotaManager,
+           replica, futureReplica, partition)
+
+    //Create the fetcher thread
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = null)
+    thread.addPartitions(Map(t1p0 -> 0))
+
+    // loop few times
+    (0 to 3).foreach { _ =>
+      thread.doWork()
+                     }
+
+    //Assert that truncate to is called exactly once (despite more loops)
+    verify(partition)
+  }
+
+  @Test
+  def shouldFetchOneReplicaAtATime(): Unit = {
+
+    //Setup all dependencies
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+    val quotaManager = createNiceMock(classOf[ReplicationQuotaManager])
+    val logManager = createMock(classOf[LogManager])
+    val replica = createNiceMock(classOf[Replica])
+    val futureReplica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+
+    //Stubs
+    expect(futureReplica.logStartOffset).andReturn(123).anyTimes()
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    stub(replica, replica, futureReplica, partition, replicaManager)
+
+    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+
+    //Create the fetcher thread
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = null)
+    thread.addPartitions(Map(t1p0 -> 0, t1p1 -> 0))
+
+    val ResultWithPartitions(fetchRequest, partitionsWithError) =
+      thread.buildFetchRequest(Seq((t1p0, new PartitionFetchState(150)), (t1p1, new PartitionFetchState(160))))
+
+    assertFalse(fetchRequest.isEmpty)
+    assertFalse(partitionsWithError.nonEmpty)
+    val request = fetchRequest.underlying.build()
+    assertEquals(0, request.minBytes)
+    val fetchInfos = request.fetchData.asScala.toSeq
+    assertEquals(1, fetchInfos.length)
+    assertEquals("Expected fetch request for largest partition", t1p1, fetchInfos.head._1)
+    assertEquals(160, fetchInfos.head._2.fetchOffset)
+  }
+
+  @Test
+  def shouldFetchNonDelayedAndNonTruncatingReplicas(): Unit = {
+
+    //Setup all dependencies
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+    val quotaManager = createNiceMock(classOf[ReplicationQuotaManager])
+    val logManager = createMock(classOf[LogManager])
+    val replica = createNiceMock(classOf[Replica])
+    val futureReplica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+
+    //Stubs
+    expect(futureReplica.logStartOffset).andReturn(123).anyTimes()
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    stub(replica, replica, futureReplica, partition, replicaManager)
+
+    replay(replicaManager, logManager, quotaManager, replica, futureReplica, partition)
+
+    //Create the fetcher thread
+    val endPoint = new BrokerEndPoint(0, "localhost", 1000)
+    val thread = new ReplicaAlterLogDirsThread(
+      "alter-logs-dirs-thread-test1",
+      sourceBroker = endPoint,
+      brokerConfig = config,
+      replicaMgr = replicaManager,
+      quota = quotaManager,
+      brokerTopicStats = null)
+    thread.addPartitions(Map(t1p0 -> 0, t1p1 -> 0))
+
+    // one partition is ready and one is truncating
+    val ResultWithPartitions(fetchRequest, partitionsWithError) =
+      thread.buildFetchRequest(Seq(
+        (t1p0, new PartitionFetchState(150)),
+        (t1p1, new PartitionFetchState(160, truncatingLog=true))))
+
+    assertFalse(fetchRequest.isEmpty)
+    assertFalse(partitionsWithError.nonEmpty)
+    val fetchInfos = fetchRequest.underlying.build().fetchData.asScala.toSeq
+    assertEquals(1, fetchInfos.length)
+    assertEquals("Expected fetch request for non-truncating partition", t1p0, fetchInfos.head._1)
+    assertEquals(150, fetchInfos.head._2.fetchOffset)
+
+    // one partition is ready and one is delayed
+    val ResultWithPartitions(fetchRequest2, partitionsWithError2) =
+      thread.buildFetchRequest(Seq(
+        (t1p0, new PartitionFetchState(140)),
+        (t1p1, new PartitionFetchState(160, delay=new DelayedItem(5000)))))
+
+    assertFalse(fetchRequest2.isEmpty)
+    assertFalse(partitionsWithError2.nonEmpty)
+    val fetchInfos2 = fetchRequest2.underlying.build().fetchData.asScala.toSeq
+    assertEquals(1, fetchInfos2.length)
+    assertEquals("Expected fetch request for non-delayed partition", t1p0, fetchInfos2.head._1)
+    assertEquals(140, fetchInfos2.head._2.fetchOffset)
+
+    // both partitions are delayed
+    val ResultWithPartitions(fetchRequest3, partitionsWithError3) =
+      thread.buildFetchRequest(Seq(
+        (t1p0, new PartitionFetchState(140, delay=new DelayedItem(5000))),
+        (t1p1, new PartitionFetchState(160, delay=new DelayedItem(5000)))))
+    assertTrue("Expected no fetch requests since all partitions are delayed", fetchRequest3.isEmpty)
+    assertFalse(partitionsWithError3.nonEmpty)
+  }
+
+  def stub(replicaT1p0: Replica, replicaT1p1: Replica, futureReplica: Replica, partition: Partition, replicaManager: ReplicaManager) = {
+    expect(replicaManager.getReplica(t1p0)).andReturn(Some(replicaT1p0)).anyTimes()
+    expect(replicaManager.getReplica(t1p0, Request.FutureLocalReplicaId)).andReturn(Some(futureReplica)).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p0)).andReturn(replicaT1p0).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p0, Request.FutureLocalReplicaId)).andReturn(futureReplica).anyTimes()
+    expect(replicaManager.getPartition(t1p0)).andReturn(Some(partition)).anyTimes()
+    expect(replicaManager.getReplica(t1p1)).andReturn(Some(replicaT1p1)).anyTimes()
+    expect(replicaManager.getReplica(t1p1, Request.FutureLocalReplicaId)).andReturn(Some(futureReplica)).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p1)).andReturn(replicaT1p1).anyTimes()
+    expect(replicaManager.getReplicaOrException(t1p1, Request.FutureLocalReplicaId)).andReturn(futureReplica).anyTimes()
+    expect(replicaManager.getPartition(t1p1)).andReturn(Some(partition)).anyTimes()
+  }
+
+  def stubWithFetchMessages(replicaT1p0: Replica, replicaT1p1: Replica, futureReplica: Replica,
+                            partition: Partition, replicaManager: ReplicaManager,
+                            responseCallback: Capture[Seq[(TopicPartition, FetchPartitionData)] => Unit]) = {
+    stub(replicaT1p0, replicaT1p1, futureReplica, partition, replicaManager)
+    expect(replicaManager.fetchMessages(
+      EasyMock.anyLong(),
+      EasyMock.anyInt(),
+      EasyMock.anyInt(),
+      EasyMock.anyInt(),
+      EasyMock.anyObject(),
+      EasyMock.anyObject(),
+      EasyMock.anyObject(),
+      EasyMock.capture(responseCallback),
+      EasyMock.anyObject()))
+      .andAnswer(new IAnswer[Unit] {
+        override def answer(): Unit = {
+          responseCallback.getValue.apply(Seq.empty[(TopicPartition, FetchPartitionData)])
+        }
+      }).anyTimes()
+  }
+}

From 6be908a8296456adee254b405605acff55fd47a5 Mon Sep 17 00:00:00 2001
From: Colin Patrick McCabe 
Date: Wed, 25 Apr 2018 17:49:02 -0700
Subject: [PATCH 0294/1847] MINOR: Refactor AdminClient ListConsumerGroups API
 (#4884)

The current Iterator-based ListConsumerGroups API is synchronous.  The API should be asynchronous to fit in with the other AdminClient APIs.  Also fix some error handling corner cases.

Reviewers: Guozhang Wang , Jason Gustafson 
---
 .../kafka/clients/admin/KafkaAdminClient.java | 160 ++++++++++--------
 .../admin/ListConsumerGroupsResult.java       | 109 ++++++------
 .../clients/admin/KafkaAdminClientTest.java   |  52 +++---
 3 files changed, 166 insertions(+), 155 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index fa3f943555bbe..d8c0bad8066eb 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -49,6 +49,7 @@
 import org.apache.kafka.common.errors.InvalidGroupIdException;
 import org.apache.kafka.common.errors.InvalidRequestException;
 import org.apache.kafka.common.errors.InvalidTopicException;
+import org.apache.kafka.common.errors.LeaderNotAvailableException;
 import org.apache.kafka.common.errors.RetriableException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.UnknownServerException;
@@ -2342,16 +2343,56 @@ void handleFailure(Throwable throwable) {
         return new DescribeConsumerGroupsResult(new HashMap>(futures));
     }
 
+    private final static class ListConsumerGroupsResults {
+        private final List errors;
+        private final HashMap listings;
+        private final HashSet remaining;
+        private final KafkaFutureImpl> future;
+
+        ListConsumerGroupsResults(Collection errors, Collection leaders,
+                                  KafkaFutureImpl> future) {
+            this.errors = new ArrayList<>(errors);
+            this.listings = new HashMap<>();
+            this.remaining = new HashSet<>(leaders);
+            this.future = future;
+            tryComplete();
+        }
+
+        synchronized void addError(Throwable throwable, Node node) {
+            ApiError error = ApiError.fromThrowable(throwable);
+            if (error.message() == null || error.message().isEmpty()) {
+                errors.add(error.error().exception(
+                    "Error listing groups on " + node));
+            } else {
+                errors.add(error.error().exception(
+                    "Error listing groups on " + node + ": " + error.message()));
+            }
+        }
+
+        synchronized void addListing(ConsumerGroupListing listing) {
+            listings.put(listing.groupId(), listing);
+        }
+
+        synchronized void tryComplete(Node leader) {
+            remaining.remove(leader);
+            tryComplete();
+        }
+
+        private synchronized void tryComplete() {
+            if (remaining.isEmpty()) {
+                ArrayList results = new ArrayList(listings.values());
+                results.addAll(errors);
+                future.complete(results);
+            }
+        }
+    };
+
     @Override
     public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions options) {
-        final Map>> futuresMap = new HashMap<>();
-        final KafkaFutureImpl> flattenFuture = new KafkaFutureImpl<>();
-        final KafkaFutureImpl listFuture = new KafkaFutureImpl<>();
-
+        final KafkaFutureImpl> all = new KafkaFutureImpl<>();
         final long nowMetadata = time.milliseconds();
         final long deadline = calcDeadlineMs(nowMetadata, options.timeoutMs());
-
-        runnable.call(new Call("listNodes", deadline, new LeastLoadedNodeProvider()) {
+        runnable.call(new Call("findGroupsMetadata", deadline, new LeastLoadedNodeProvider()) {
             @Override
             AbstractRequest.Builder createRequest(int timeoutMs) {
                 return new MetadataRequest.Builder(Collections.singletonList(Topic.GROUP_METADATA_TOPIC_NAME), true);
@@ -2360,68 +2401,38 @@ AbstractRequest.Builder createRequest(int timeoutMs) {
             @Override
             void handleResponse(AbstractResponse abstractResponse) {
                 MetadataResponse metadataResponse = (MetadataResponse) abstractResponse;
-
+                final List metadataExceptions = new ArrayList<>();
+                final HashSet leaders = new HashSet<>();
                 for (final MetadataResponse.TopicMetadata metadata : metadataResponse.topicMetadata()) {
-                    if (metadata.topic().equals(Topic.GROUP_METADATA_TOPIC_NAME)) {
+                    if (metadata.error() != Errors.NONE) {
+                        metadataExceptions.add(metadata.error().exception("Unable to locate " +
+                            Topic.GROUP_METADATA_TOPIC_NAME));
+                    } else if (!metadata.topic().equals(Topic.GROUP_METADATA_TOPIC_NAME)) {
+                        metadataExceptions.add(new UnknownServerException("Server returned unrequested " +
+                            "information about unexpected topic " + metadata.topic()));
+                    } else {
                         for (final MetadataResponse.PartitionMetadata partitionMetadata : metadata.partitionMetadata()) {
                             final Node leader = partitionMetadata.leader();
                             if (partitionMetadata.error() != Errors.NONE) {
                                 // TODO: KAFKA-6789, retry based on the error code
-                                KafkaFutureImpl> future = new KafkaFutureImpl<>();
-                                future.completeExceptionally(partitionMetadata.error().exception());
-                                // if it is the leader not found error, then the leader might be NoNode; if there are more than
-                                // one such error, we will only have one entry in the map. For now it is okay since we are not
-                                // guaranteeing to return the full list of consumers still.
-                                futuresMap.put(leader, future);
+                                metadataExceptions.add(partitionMetadata.error().exception("Unable to find " +
+                                    "leader for partition " + partitionMetadata.partition() + " of " +
+                                    Topic.GROUP_METADATA_TOPIC_NAME));
+                            } else if (leader == null || leader.equals(Node.noNode())) {
+                                metadataExceptions.add(new LeaderNotAvailableException("Unable to find leader " +
+                                    "for partition " + partitionMetadata.partition() + " of " +
+                                    Topic.GROUP_METADATA_TOPIC_NAME));
                             } else {
-                                futuresMap.put(leader, new KafkaFutureImpl>());
+                                leaders.add(leader);
                             }
                         }
-                        listFuture.complete(null);
-                    } else {
-                        if (metadata.error() != Errors.NONE)
-                            listFuture.completeExceptionally(metadata.error().exception());
-                        else
-                            listFuture.completeExceptionally(new IllegalStateException("Unexpected topic metadata for "
-                                    + metadata.topic() + " is returned; cannot find the brokers to query consumer listings."));
                     }
                 }
-
-                // we have to flatten the future here instead in the result, because we need to wait until the map of nodes
-                // are known from the listNode request.
-                flattenFuture.copyWith(
-                        KafkaFuture.allOf(futuresMap.values().toArray(new KafkaFuture[0])),
-                        new KafkaFuture.BaseFunction>() {
-                            @Override
-                            public Collection apply(Void v) {
-                                List listings = new ArrayList<>();
-                                for (Map.Entry>> entry : futuresMap.entrySet()) {
-                                    Collection results;
-                                    try {
-                                        results = entry.getValue().get();
-                                        listings.addAll(results);
-                                    } catch (Throwable e) {
-                                        // This should be unreachable, because allOf ensured that all the futures
-                                        // completed successfully.
-                                        throw new RuntimeException(e);
-                                    }
-                                }
-                                return listings;
-                            }
-                        });
-
-                for (final Map.Entry>> entry : futuresMap.entrySet()) {
-                    // skip sending the request for those futures who have already failed
-                    if (entry.getValue().isCompletedExceptionally())
-                        continue;
-
+                final ListConsumerGroupsResults results =
+                    new ListConsumerGroupsResults(metadataExceptions, leaders, all);
+                for (final Node node : leaders) {
                     final long nowList = time.milliseconds();
-
-                    final int brokerId = entry.getKey().id();
-                    final KafkaFutureImpl> future = entry.getValue();
-
-                    runnable.call(new Call("listConsumerGroups", deadline, new ConstantNodeIdProvider(brokerId)) {
-
+                    runnable.call(new Call("listConsumerGroups", deadline, new ConstantNodeIdProvider(node.id())) {
                         @Override
                         AbstractRequest.Builder createRequest(int timeoutMs) {
                             return new ListGroupsRequest.Builder();
@@ -2430,39 +2441,42 @@ AbstractRequest.Builder createRequest(int timeoutMs) {
                         @Override
                         void handleResponse(AbstractResponse abstractResponse) {
                             final ListGroupsResponse response = (ListGroupsResponse) abstractResponse;
-
-                            if (response.error() != Errors.NONE) {
-                                future.completeExceptionally(response.error().exception());
-                            } else {
-                                final List groupsListing = new ArrayList<>();
-                                for (ListGroupsResponse.Group group : response.groups()) {
-                                    if (group.protocolType().equals(ConsumerProtocol.PROTOCOL_TYPE) || group.protocolType().isEmpty()) {
-                                        final String groupId = group.groupId();
-                                        final String protocolType = group.protocolType();
-                                        final ConsumerGroupListing groupListing = new ConsumerGroupListing(groupId, protocolType.isEmpty());
-                                        groupsListing.add(groupListing);
+                            synchronized (results) {
+                                if (response.error() != Errors.NONE) {
+                                    results.addError(response.error().exception(), node);
+                                } else {
+                                    for (ListGroupsResponse.Group group : response.groups()) {
+                                        if (group.protocolType().equals(ConsumerProtocol.PROTOCOL_TYPE) ||
+                                            group.protocolType().isEmpty()) {
+                                            final String groupId = group.groupId();
+                                            final String protocolType = group.protocolType();
+                                            final ConsumerGroupListing groupListing = new ConsumerGroupListing(groupId, protocolType.isEmpty());
+                                            results.addListing(groupListing);
+                                        }
                                     }
                                 }
-                                future.complete(groupsListing);
+                                results.tryComplete(node);
                             }
                         }
 
                         @Override
                         void handleFailure(Throwable throwable) {
-                            future.completeExceptionally(throwable);
+                            synchronized (results) {
+                                results.addError(throwable, node);
+                                results.tryComplete(node);
+                            }
                         }
                     }, nowList);
-
                 }
             }
 
             @Override
             void handleFailure(Throwable throwable) {
-                listFuture.completeExceptionally(throwable);
+                all.complete(Collections.singletonList(throwable));
             }
         }, nowMetadata);
 
-        return new ListConsumerGroupsResult(listFuture, flattenFuture, futuresMap);
+        return new ListConsumerGroupsResult(all);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java
index c3f1236eb0915..0ac852945f64e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java
@@ -18,14 +18,11 @@
 package org.apache.kafka.clients.admin;
 
 import org.apache.kafka.common.KafkaFuture;
-import org.apache.kafka.common.Node;
 import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.internals.KafkaFutureImpl;
-import org.apache.kafka.common.utils.AbstractIterator;
 
+import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
 
 /**
  * The result of the {@link AdminClient#listConsumerGroups()} call.
@@ -34,70 +31,72 @@
  */
 @InterfaceStability.Evolving
 public class ListConsumerGroupsResult {
-    private final Map>> futuresMap;
-    private final KafkaFuture> flattenFuture;
-    private final KafkaFuture listFuture;
+    private final KafkaFutureImpl> all;
+    private final KafkaFutureImpl> valid;
+    private final KafkaFutureImpl> errors;
 
-    ListConsumerGroupsResult(final KafkaFuture listFuture,
-                             final KafkaFuture> flattenFuture,
-                             final Map>> futuresMap) {
-        this.flattenFuture = flattenFuture;
-        this.listFuture = listFuture;
-        this.futuresMap = futuresMap;
-    }
-
-    private class FutureConsumerGroupListingIterator extends AbstractIterator> {
-        private Iterator>> futuresIter;
-        private Iterator innerIter;
-
-        @Override
-        protected KafkaFuture makeNext() {
-            if (futuresIter == null) {
-                try {
-                    listFuture.get();
-                } catch (Exception e) {
-                    // the list future has failed, there will be no listings to show at all
-                    return allDone();
-                }
-
-                futuresIter = futuresMap.values().iterator();
-            }
-
-            while (innerIter == null || !innerIter.hasNext()) {
-                if (futuresIter.hasNext()) {
-                    KafkaFuture> collectionFuture = futuresIter.next();
-                    try {
-                        Collection collection = collectionFuture.get();
-                        innerIter = collection.iterator();
-                    } catch (Exception e) {
-                        KafkaFutureImpl future = new KafkaFutureImpl<>();
-                        future.completeExceptionally(e);
-                        return future;
+    ListConsumerGroupsResult(KafkaFutureImpl> future) {
+        this.all = new KafkaFutureImpl<>();
+        this.valid = new KafkaFutureImpl<>();
+        this.errors = new KafkaFutureImpl<>();
+        future.thenApply(new KafkaFuture.BaseFunction, Void>() {
+            @Override
+            public Void apply(Collection results) {
+                ArrayList curErrors = new ArrayList<>();
+                ArrayList curValid = new ArrayList<>();
+                for (Object resultObject : results) {
+                    if (resultObject instanceof Throwable) {
+                        curErrors.add((Throwable) resultObject);
+                    } else {
+                        curValid.add((ConsumerGroupListing) resultObject);
                     }
+                }
+                if (!curErrors.isEmpty()) {
+                    all.completeExceptionally(curErrors.get(0));
                 } else {
-                    return allDone();
+                    all.complete(curValid);
                 }
+                valid.complete(curValid);
+                errors.complete(curErrors);
+                return null;
             }
+        });
+    }
 
-            KafkaFutureImpl future = new KafkaFutureImpl<>();
-            future.complete(innerIter.next());
-            return future;
-        }
+    /**
+     * Returns a future that yields either an exception, or the full set of consumer group
+     * listings.
+     *
+     * In the event of a failure, the future yields nothing but the first exception which
+     * occurred.
+     */
+    public KafkaFutureImpl> all() {
+        return all;
     }
 
     /**
-     * Return an iterator of futures for ConsumerGroupListing objects; the returned future will throw exception
-     * if we cannot get a complete collection of consumer listings.
+     * Returns a future which yields just the valid listings.
+     *
+     * This future never fails with an error, no matter what happens.  Errors are completely
+     * ignored.  If nothing can be fetched, an empty collection is yielded.
+     * If there is an error, but some results can be returned, this future will yield
+     * those partial results.  When using this future, it is a good idea to also check
+     * the errors future so that errors can be displayed and handled.
      */
-    public Iterator> iterator() {
-        return new FutureConsumerGroupListingIterator();
+    public KafkaFutureImpl> valid() {
+        return valid;
     }
 
     /**
-     * Return a future which yields a full collection of ConsumerGroupListing objects; will throw exception
-     * if we cannot get a complete collection of consumer listings.
+     * Returns a future which yields just the errors which occurred.
+     *
+     * If this future yields a non-empty collection, it is very likely that elements are
+     * missing from the valid() set.
+     *
+     * This future itself never fails with an error.  In the event of an error, this future
+     * will successfully yield a collection containing at least one exception.
      */
-    public KafkaFuture> all() {
-        return flattenFuture;
+    public KafkaFutureImpl> errors() {
+        return errors;
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index d2789b626211a..0debed3dead58 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.common.acl.AclPermissionType;
 import org.apache.kafka.common.config.ConfigResource;
 import org.apache.kafka.common.errors.AuthenticationException;
+import org.apache.kafka.common.errors.CoordinatorNotAvailableException;
 import org.apache.kafka.common.errors.InvalidTopicException;
 import org.apache.kafka.common.errors.LeaderNotAvailableException;
 import org.apache.kafka.common.errors.NotLeaderForPartitionException;
@@ -80,7 +81,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -648,9 +648,8 @@ public void testDeleteRecords() throws Exception {
         }
     }
 
-    //Ignoring test to be fixed on follow-up PR
     @Test
-    public void testListConsumerGroups() {
+    public void testListConsumerGroups() throws Exception {
         final HashMap nodes = new HashMap<>();
         Node node0 = new Node(0, "localhost", 8121);
         Node node1 = new Node(1, "localhost", 8122);
@@ -685,7 +684,8 @@ public void testListConsumerGroups() {
                             env.cluster().nodes(),
                             env.cluster().clusterResource().clusterId(),
                             env.cluster().controller().id(),
-                            Collections.singletonList(new MetadataResponse.TopicMetadata(Errors.NONE, Topic.GROUP_METADATA_TOPIC_NAME, true, partitionMetadata))));
+                            Collections.singletonList(new MetadataResponse.TopicMetadata(Errors.NONE,
+                                Topic.GROUP_METADATA_TOPIC_NAME, true, partitionMetadata))));
 
             env.kafkaClient().prepareResponseFrom(
                     new ListGroupsResponse(
@@ -713,31 +713,29 @@ public void testListConsumerGroups() {
                     node2);
 
             final ListConsumerGroupsResult result = env.adminClient().listConsumerGroups();
-
-            try {
-                Collection listing = result.all().get();
-                fail("Expected to throw exception");
-            } catch (Exception e) {
-                // this is good
-            }
-
-            Iterator> iterator = result.iterator();
-            int numListing = 0;
-            int numFailure = 0;
-
-            while (iterator.hasNext()) {
-                KafkaFuture future = iterator.next();
-                try {
-                    ConsumerGroupListing listing = future.get();
-                    numListing++;
-                    assertTrue(listing.groupId().equals("group-1") || listing.groupId().equals("group-2"));
-                } catch (Exception e) {
-                    numFailure++;
-                }
+            assertFutureError(result.all(), CoordinatorNotAvailableException.class);
+            Collection listings = result.valid().get();
+            assertEquals(2, listings.size());
+            for (ConsumerGroupListing listing : listings) {
+                assertTrue(listing.groupId().equals("group-1") || listing.groupId().equals("group-2"));
             }
+            assertEquals(1, result.errors().get().size());
 
-            assertEquals(2, numListing);
-            assertEquals(1, numFailure);
+            // Test handling the error where we are unable to get metadata for the __consumer_offsets topic.
+            env.kafkaClient().prepareResponse(
+                new MetadataResponse(
+                    env.cluster().nodes(),
+                    env.cluster().clusterResource().clusterId(),
+                    env.cluster().controller().id(),
+                    Collections.singletonList(new MetadataResponse.TopicMetadata(
+                        Errors.UNKNOWN_TOPIC_OR_PARTITION, Topic.GROUP_METADATA_TOPIC_NAME,
+                        true, Collections.emptyList()))));
+            final ListConsumerGroupsResult result2 = env.adminClient().listConsumerGroups();
+            Collection errors = result2.errors().get();
+            assertEquals(1, errors.size());
+            assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.forException(errors.iterator().next()));
+            assertTrue(result2.valid().get().isEmpty());
+            assertFutureError(result2.all(), UnknownTopicOrPartitionException.class);
         }
     }
 

From 459efb02ad243e87f42caffcf4f3a0004c41ccd6 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Thu, 26 Apr 2018 06:58:36 -0700
Subject: [PATCH 0295/1847] HOTFIX: ListConsumerGroupsResult should use
 KafkaFuture (#4933)

---
 .../kafka/clients/admin/ListConsumerGroupsResult.java       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java
index 0ac852945f64e..7de485bd2b989 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java
@@ -70,7 +70,7 @@ public Void apply(Collection results) {
      * In the event of a failure, the future yields nothing but the first exception which
      * occurred.
      */
-    public KafkaFutureImpl> all() {
+    public KafkaFuture> all() {
         return all;
     }
 
@@ -83,7 +83,7 @@ public KafkaFutureImpl> all() {
      * those partial results.  When using this future, it is a good idea to also check
      * the errors future so that errors can be displayed and handled.
      */
-    public KafkaFutureImpl> valid() {
+    public KafkaFuture> valid() {
         return valid;
     }
 
@@ -96,7 +96,7 @@ public KafkaFutureImpl> valid() {
      * This future itself never fails with an error.  In the event of an error, this future
      * will successfully yield a collection containing at least one exception.
      */
-    public KafkaFutureImpl> errors() {
+    public KafkaFuture> errors() {
         return errors;
     }
 }

From fc6e92260c3ff3b5ef72ac05e7a518a8cb2e7090 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Thu, 26 Apr 2018 12:01:17 -0500
Subject: [PATCH 0296/1847] MINOR: fix NamedCache metrics in Streams (#4917)

* Fixes a bug in which all NamedCache instances in a process shared
one parent metric.

* Also fixes a bug which incorrectly computed the per-cache metric tag
(which was undetected due to the former bug).

* Drop the StreamsMetricsConventions#xLevelSensorName convention
in favor of StreamsMetricsImpl#xLevelSensor to allow StreamsMetricsImpl
to track thread- and cache-level metrics, so that they may be cleanly declared
from anywhere but still unloaded at the appropriate time. This was necessary
right now so that the NamedCache could register a thread-level parent sensor
to be unloaded when the thread, not the cache, is closed.

* The above changes made it mostly unnecessary for the StreamsMetricsImpl to
expose a reference to the underlying Metrics registry, so I did a little extra work
to remove that reference, including removing inconsistently-used and unnecessary
calls to Metrics#close() in the tests.

The existing tests should be sufficient to verify this change.

Reviewers: Bill Bejeck , Guozhang Wang 
---
 .../apache/kafka/streams/KafkaStreams.java    |   3 +-
 .../internals/GlobalStreamThread.java         |   2 +-
 .../processor/internals/StreamTask.java       |  53 +++++++-
 .../processor/internals/StreamThread.java     |  82 +++++-------
 .../metrics/StreamsMetricsConventions.java    |  39 ------
 .../internals/metrics/StreamsMetricsImpl.java | 117 +++++++++++++++---
 .../streams/state/internals/NamedCache.java   |  83 ++++++++-----
 .../streams/state/internals/ThreadCache.java  |  10 +-
 ...amSessionWindowAggregateProcessorTest.java |   1 -
 .../internals/ProcessorNodeTest.java          |   1 -
 .../processor/internals/SinkNodeTest.java     |   6 -
 .../processor/internals/StreamTaskTest.java   |   1 -
 .../processor/internals/StreamThreadTest.java |   9 --
 .../internals/AbstractKeyValueStoreTest.java  |   1 -
 .../internals/CachingKeyValueStoreTest.java   |   1 -
 .../internals/CachingSessionStoreTest.java    |   1 -
 .../internals/CachingWindowStoreTest.java     |   1 -
 .../ChangeLoggingKeyValueBytesStoreTest.java  |   1 -
 .../internals/MeteredWindowStoreTest.java     |   6 -
 .../state/internals/NamedCacheTest.java       |  25 ++--
 .../RocksDBKeyValueStoreSupplierTest.java     |   1 -
 .../RocksDBSegmentedBytesStoreTest.java       |   1 -
 .../RocksDBSessionStoreSupplierTest.java      |   1 -
 .../internals/RocksDBSessionStoreTest.java    |   1 -
 .../state/internals/RocksDBStoreTest.java     |   1 -
 .../RocksDBWindowStoreSupplierTest.java       |   1 -
 .../internals/RocksDBWindowStoreTest.java     |   1 -
 .../state/internals/SegmentIteratorTest.java  |   1 -
 .../streams/state/internals/SegmentsTest.java |   1 -
 .../internals/StoreChangeLoggerTest.java      |   6 -
 .../test/InternalMockProcessorContext.java    |   6 -
 .../apache/kafka/test/KStreamTestDriver.java  |   8 +-
 32 files changed, 258 insertions(+), 214 deletions(-)
 delete mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsConventions.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index a99f8cc19634c..776dde708164e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -673,7 +673,8 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder,
             throw new StreamsException(fatal);
         }
 
-        final MetricConfig metricConfig = new MetricConfig().samples(config.getInt(StreamsConfig.METRICS_NUM_SAMPLES_CONFIG))
+        final MetricConfig metricConfig = new MetricConfig()
+            .samples(config.getInt(StreamsConfig.METRICS_NUM_SAMPLES_CONFIG))
             .recordLevel(Sensor.RecordingLevel.forName(config.getString(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG)))
             .timeWindow(config.getLong(StreamsConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS);
         final List reporters = config.getConfiguredInstances(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
index af3c7dbc3047b..1c348975f2acb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/GlobalStreamThread.java
@@ -298,7 +298,7 @@ public void run() {
                 log.error("Failed to close state maintainer due to the following error:", e);
             }
 
-            streamsMetrics.removeOwnedSensors();
+            streamsMetrics.removeAllThreadLevelSensors();
 
             setState(DEAD);
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index d9515b15ecd41..b9753249eaeed 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -22,9 +22,14 @@
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.metrics.stats.Avg;
+import org.apache.kafka.common.metrics.stats.Count;
+import org.apache.kafka.common.metrics.stats.Max;
+import org.apache.kafka.common.metrics.stats.Rate;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.DeserializationExceptionHandler;
@@ -42,6 +47,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static java.lang.String.format;
 import static java.util.Collections.singleton;
@@ -72,16 +78,57 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator
     protected static final class TaskMetrics {
         final StreamsMetricsImpl metrics;
         final Sensor taskCommitTimeSensor;
+        private final String taskName;
 
 
         TaskMetrics(final TaskId id, final StreamsMetricsImpl metrics) {
-            final String name = id.toString();
+            taskName = id.toString();
             this.metrics = metrics;
-            taskCommitTimeSensor = metrics.addLatencyAndThroughputSensor("task", name, "commit", Sensor.RecordingLevel.DEBUG);
+            final String group = "stream-task-metrics";
+
+            // first add the global operation metrics if not yet, with the global tags only
+            final Map allTagMap = metrics.tagMap("task-id", "all");
+            final Sensor parent = metrics.threadLevelSensor("commit", Sensor.RecordingLevel.DEBUG);
+            parent.add(
+                new MetricName("commit-latency-avg", group, "The average latency of commit operation.", allTagMap),
+                new Avg()
+            );
+            parent.add(
+                new MetricName("commit-latency-max", group, "The max latency of commit operation.", allTagMap),
+                new Max()
+            );
+            parent.add(
+                new MetricName("commit-rate", group, "The average number of occurrence of commit operation per second.", allTagMap),
+                new Rate(TimeUnit.SECONDS, new Count())
+            );
+            parent.add(
+                new MetricName("commit-total", group, "The total number of occurrence of commit operations.", allTagMap),
+                new Count()
+            );
+
+            // add the operation metrics with additional tags
+            final Map tagMap = metrics.tagMap("task-id", taskName);
+            taskCommitTimeSensor = metrics.taskLevelSensor("commit", taskName, Sensor.RecordingLevel.DEBUG, parent);
+            taskCommitTimeSensor.add(
+                new MetricName("commit-latency-avg", group, "The average latency of commit operation.", tagMap),
+                new Avg()
+            );
+            taskCommitTimeSensor.add(
+                new MetricName("commit-latency-max", group, "The max latency of commit operation.", tagMap),
+                new Max()
+            );
+            taskCommitTimeSensor.add(
+                new MetricName("commit-rate", group, "The average number of occurrence of commit operation per second.", tagMap),
+                new Rate(TimeUnit.SECONDS, new Count())
+            );
+            taskCommitTimeSensor.add(
+                new MetricName("commit-total", group, "The total number of occurrence of commit operations.", tagMap),
+                new Count()
+            );
         }
 
         void removeAllSensors() {
-            metrics.removeSensor(taskCommitTimeSensor);
+            metrics.removeAllTaskLevelSensors(taskName);
         }
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 39727be5a3ea1..e4ad1385e5cf2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -52,10 +52,8 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -64,7 +62,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.Collections.singleton;
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsConventions.threadLevelSensorName;
 
 public class StreamThread extends Thread {
 
@@ -509,58 +506,41 @@ static class StreamsMetricsThreadImpl extends StreamsMetricsImpl {
         private final Sensor taskCreatedSensor;
         private final Sensor tasksClosedSensor;
 
-        private final Deque ownedSensors = new LinkedList<>();
-
         StreamsMetricsThreadImpl(final Metrics metrics, final String threadName) {
             super(metrics, threadName);
-            final String groupName = "stream-metrics";
-
-            commitTimeSensor = metrics.sensor(threadLevelSensorName(threadName, "commit-latency"), Sensor.RecordingLevel.INFO);
-            commitTimeSensor.add(metrics.metricName("commit-latency-avg", groupName, "The average commit time in ms", tags()), new Avg());
-            commitTimeSensor.add(metrics.metricName("commit-latency-max", groupName, "The maximum commit time in ms", tags()), new Max());
-            commitTimeSensor.add(metrics.metricName("commit-rate", groupName, "The average per-second number of commit calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
-            commitTimeSensor.add(metrics.metricName("commit-total", groupName, "The total number of commit calls", tags()), new Count());
-            ownedSensors.push(commitTimeSensor.name());
-
-            pollTimeSensor = metrics.sensor(threadLevelSensorName(threadName, "poll-latency"), Sensor.RecordingLevel.INFO);
-            pollTimeSensor.add(metrics.metricName("poll-latency-avg", groupName, "The average poll time in ms", tags()), new Avg());
-            pollTimeSensor.add(metrics.metricName("poll-latency-max", groupName, "The maximum poll time in ms", tags()), new Max());
-            pollTimeSensor.add(metrics.metricName("poll-rate", groupName, "The average per-second number of record-poll calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
-            pollTimeSensor.add(metrics.metricName("poll-total", groupName, "The total number of record-poll calls", tags()), new Count());
-            ownedSensors.push(pollTimeSensor.name());
-
-            processTimeSensor = metrics.sensor(threadLevelSensorName(threadName, "process-latency"), Sensor.RecordingLevel.INFO);
-            processTimeSensor.add(metrics.metricName("process-latency-avg", groupName, "The average process time in ms", tags()), new Avg());
-            processTimeSensor.add(metrics.metricName("process-latency-max", groupName, "The maximum process time in ms", tags()), new Max());
-            processTimeSensor.add(metrics.metricName("process-rate", groupName, "The average per-second number of process calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
-            processTimeSensor.add(metrics.metricName("process-total", groupName, "The total number of process calls", tags()), new Count());
-            ownedSensors.push(processTimeSensor.name());
-
-            punctuateTimeSensor = metrics.sensor(threadLevelSensorName(threadName, "punctuate-latency"), Sensor.RecordingLevel.INFO);
-            punctuateTimeSensor.add(metrics.metricName("punctuate-latency-avg", groupName, "The average punctuate time in ms", tags()), new Avg());
-            punctuateTimeSensor.add(metrics.metricName("punctuate-latency-max", groupName, "The maximum punctuate time in ms", tags()), new Max());
-            punctuateTimeSensor.add(metrics.metricName("punctuate-rate", groupName, "The average per-second number of punctuate calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
-            punctuateTimeSensor.add(metrics.metricName("punctuate-total", groupName, "The total number of punctuate calls", tags()), new Count());
-            ownedSensors.push(punctuateTimeSensor.name());
-
-            taskCreatedSensor = metrics.sensor(threadLevelSensorName(threadName, "task-created"), Sensor.RecordingLevel.INFO);
+            final String group = "stream-metrics";
+
+            commitTimeSensor = threadLevelSensor("commit-latency", Sensor.RecordingLevel.INFO);
+            commitTimeSensor.add(metrics.metricName("commit-latency-avg", group, "The average commit time in ms", tags()), new Avg());
+            commitTimeSensor.add(metrics.metricName("commit-latency-max", group, "The maximum commit time in ms", tags()), new Max());
+            commitTimeSensor.add(metrics.metricName("commit-rate", group, "The average per-second number of commit calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
+            commitTimeSensor.add(metrics.metricName("commit-total", group, "The total number of commit calls", tags()), new Count());
+
+            pollTimeSensor = threadLevelSensor("poll-latency", Sensor.RecordingLevel.INFO);
+            pollTimeSensor.add(metrics.metricName("poll-latency-avg", group, "The average poll time in ms", tags()), new Avg());
+            pollTimeSensor.add(metrics.metricName("poll-latency-max", group, "The maximum poll time in ms", tags()), new Max());
+            pollTimeSensor.add(metrics.metricName("poll-rate", group, "The average per-second number of record-poll calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
+            pollTimeSensor.add(metrics.metricName("poll-total", group, "The total number of record-poll calls", tags()), new Count());
+
+            processTimeSensor = threadLevelSensor("process-latency", Sensor.RecordingLevel.INFO);
+            processTimeSensor.add(metrics.metricName("process-latency-avg", group, "The average process time in ms", tags()), new Avg());
+            processTimeSensor.add(metrics.metricName("process-latency-max", group, "The maximum process time in ms", tags()), new Max());
+            processTimeSensor.add(metrics.metricName("process-rate", group, "The average per-second number of process calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
+            processTimeSensor.add(metrics.metricName("process-total", group, "The total number of process calls", tags()), new Count());
+
+            punctuateTimeSensor = threadLevelSensor("punctuate-latency", Sensor.RecordingLevel.INFO);
+            punctuateTimeSensor.add(metrics.metricName("punctuate-latency-avg", group, "The average punctuate time in ms", tags()), new Avg());
+            punctuateTimeSensor.add(metrics.metricName("punctuate-latency-max", group, "The maximum punctuate time in ms", tags()), new Max());
+            punctuateTimeSensor.add(metrics.metricName("punctuate-rate", group, "The average per-second number of punctuate calls", tags()), new Rate(TimeUnit.SECONDS, new Count()));
+            punctuateTimeSensor.add(metrics.metricName("punctuate-total", group, "The total number of punctuate calls", tags()), new Count());
+
+            taskCreatedSensor = threadLevelSensor("task-created", Sensor.RecordingLevel.INFO);
             taskCreatedSensor.add(metrics.metricName("task-created-rate", "stream-metrics", "The average per-second number of newly created tasks", tags()), new Rate(TimeUnit.SECONDS, new Count()));
             taskCreatedSensor.add(metrics.metricName("task-created-total", "stream-metrics", "The total number of newly created tasks", tags()), new Total());
-            ownedSensors.push(taskCreatedSensor.name());
-
-            tasksClosedSensor = metrics.sensor(threadLevelSensorName(threadName, "task-closed"), Sensor.RecordingLevel.INFO);
-            tasksClosedSensor.add(metrics.metricName("task-closed-rate", groupName, "The average per-second number of closed tasks", tags()), new Rate(TimeUnit.SECONDS, new Count()));
-            tasksClosedSensor.add(metrics.metricName("task-closed-total", groupName, "The total number of closed tasks", tags()), new Total());
-            ownedSensors.push(tasksClosedSensor.name());
-        }
 
-        public void removeOwnedSensors() {
-            synchronized (ownedSensors) {
-                super.removeOwnedSensors();
-                while (!ownedSensors.isEmpty()) {
-                    registry().removeSensor(ownedSensors.pop());
-                }
-            }
+            tasksClosedSensor = threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
+            tasksClosedSensor.add(metrics.metricName("task-closed-rate", group, "The average per-second number of closed tasks", tags()), new Rate(TimeUnit.SECONDS, new Count()));
+            tasksClosedSensor.add(metrics.metricName("task-closed-total", group, "The total number of closed tasks", tags()), new Total());
         }
     }
 
@@ -1165,7 +1145,7 @@ private void completeShutdown(final boolean cleanRun) {
         } catch (final Throwable e) {
             log.error("Failed to close restore consumer due to the following error:", e);
         }
-        streamsMetrics.removeOwnedSensors();
+        streamsMetrics.removeAllThreadLevelSensors();
 
         setState(State.DEAD);
         log.info("Shutdown complete");
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsConventions.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsConventions.java
deleted file mode 100644
index cfe206c7d9cd4..0000000000000
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsConventions.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.processor.internals.metrics;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public final class StreamsMetricsConventions {
-    private StreamsMetricsConventions() {
-    }
-
-    public static String threadLevelSensorName(final String threadName, final String sensorName) {
-        return "thread." + threadName + "." + sensorName;
-    }
-
-    static Map threadLevelTags(final String threadName, final Map tags) {
-        if (tags.containsKey("client-id")) {
-            return tags;
-        } else {
-            final LinkedHashMap newTags = new LinkedHashMap<>(tags);
-            newTags.put("client-id", threadName);
-            return newTags;
-        }
-    }
-}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 76a1d2b1dabea..02512656bd016 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -31,35 +31,125 @@
 import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
-import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsConventions.threadLevelSensorName;
-
 public class StreamsMetricsImpl implements StreamsMetrics {
     private final Metrics metrics;
     private final Map tags;
     private final Map parentSensors;
-    private final Deque ownedSensors = new LinkedList<>();
     private final Sensor skippedRecordsSensor;
+    private final String threadName;
+
+    private final Deque threadLevelSensors = new LinkedList<>();
+    private final Map> taskLevelSensors = new HashMap<>();
+    private final Map> cacheLevelSensors = new HashMap<>();
 
     public StreamsMetricsImpl(final Metrics metrics, final String threadName) {
         Objects.requireNonNull(metrics, "Metrics cannot be null");
+        this.threadName = threadName;
 
         this.metrics = metrics;
-        this.tags = StreamsMetricsConventions.threadLevelTags(threadName, Collections.emptyMap());
+
+
+        final HashMap tags = new LinkedHashMap<>();
+        tags.put("client-id", threadName);
+        this.tags = Collections.unmodifiableMap(tags);
+
         this.parentSensors = new HashMap<>();
 
-        skippedRecordsSensor = metrics.sensor(threadLevelSensorName(threadName, "skipped-records"), Sensor.RecordingLevel.INFO);
-        skippedRecordsSensor.add(metrics.metricName("skipped-records-rate", "stream-metrics", "The average per-second number of skipped records", tags), new Rate(TimeUnit.SECONDS, new Count()));
-        skippedRecordsSensor.add(metrics.metricName("skipped-records-total", "stream-metrics", "The total number of skipped records", tags), new Total());
-        ownedSensors.push(skippedRecordsSensor.name());
+        final String group = "stream-metrics";
+        skippedRecordsSensor = threadLevelSensor("skipped-records", Sensor.RecordingLevel.INFO);
+        skippedRecordsSensor.add(metrics.metricName("skipped-records-rate", group, "The average per-second number of skipped records", tags), new Rate(TimeUnit.SECONDS, new Count()));
+        skippedRecordsSensor.add(metrics.metricName("skipped-records-total", group, "The total number of skipped records", tags), new Total());
+    }
+
+    public final Sensor threadLevelSensor(final String sensorName,
+                                          final Sensor.RecordingLevel recordingLevel,
+                                          final Sensor... parents) {
+        synchronized (threadLevelSensors) {
+            final String fullSensorName = threadName + "." + sensorName;
+            final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
+            threadLevelSensors.push(fullSensorName);
+
+            return sensor;
+        }
+    }
+
+    public final void removeAllThreadLevelSensors() {
+        synchronized (threadLevelSensors) {
+            while (!threadLevelSensors.isEmpty()) {
+                metrics.removeSensor(threadLevelSensors.pop());
+            }
+        }
+    }
+
+    public final Sensor taskLevelSensor(final String taskName,
+                                         final String sensorName,
+                                         final Sensor.RecordingLevel recordingLevel,
+                                         final Sensor... parents) {
+        final String key = threadName + "." + taskName;
+        synchronized (taskLevelSensors) {
+            if (!taskLevelSensors.containsKey(key)) {
+                taskLevelSensors.put(key, new LinkedList());
+            }
+
+            final String fullSensorName = key + "." + sensorName;
+
+            final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
+
+            taskLevelSensors.get(key).push(fullSensorName);
+
+            return sensor;
+        }
     }
 
-    public final Metrics registry() {
-        return metrics;
+    public final void removeAllTaskLevelSensors(final String taskName) {
+        final String key = threadName + "." + taskName;
+        synchronized (taskLevelSensors) {
+            if (taskLevelSensors.containsKey(key)) {
+                while (!taskLevelSensors.get(key).isEmpty()) {
+                    metrics.removeSensor(taskLevelSensors.get(key).pop());
+                }
+                taskLevelSensors.remove(key);
+            }
+        }
+    }
+
+    public final Sensor cacheLevelSensor(final String taskName,
+                                         final String cacheName,
+                                         final String sensorName,
+                                         final Sensor.RecordingLevel recordingLevel,
+                                         final Sensor... parents) {
+        final String key = threadName + "." + taskName + "." + cacheName;
+        synchronized (cacheLevelSensors) {
+            if (!cacheLevelSensors.containsKey(key)) {
+                cacheLevelSensors.put(key, new LinkedList());
+            }
+
+            final String fullSensorName = key + "." + sensorName;
+
+            final Sensor sensor = metrics.sensor(fullSensorName, recordingLevel, parents);
+
+            cacheLevelSensors.get(key).push(fullSensorName);
+
+            return sensor;
+        }
+    }
+
+    public final void removeAllCacheLevelSensors(final String taskName, final String cacheName) {
+        final String key = threadName + "." + taskName + "." + cacheName;
+        synchronized (cacheLevelSensors) {
+            if (cacheLevelSensors.containsKey(key)) {
+                while (!cacheLevelSensors.get(key).isEmpty()) {
+                    metrics.removeSensor(cacheLevelSensors.get(key).pop());
+                }
+                cacheLevelSensors.remove(key);
+            }
+        }
     }
 
     protected final Map tags() {
@@ -236,11 +326,4 @@ public void removeSensor(final Sensor sensor) {
         }
     }
 
-    public void removeOwnedSensors() {
-        synchronized (ownedSensors) {
-            while (!ownedSensors.isEmpty()) {
-                metrics.removeSensor(ownedSensors.pop());
-            }
-        }
-    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index de62a2d3696fc..d058c9cee4fa6 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -16,13 +16,13 @@
  */
 package org.apache.kafka.streams.state.internals;
 
+import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Min;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.StreamsMetrics;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -53,7 +53,7 @@ class NamedCache {
     private long numOverwrites = 0;
     private long numFlushes = 0;
 
-    NamedCache(final String name, final StreamsMetrics metrics) {
+    NamedCache(final String name, final StreamsMetricsImpl metrics) {
         this.name = name;
         this.namedCacheMetrics = new NamedCacheMetrics(metrics, name);
     }
@@ -355,45 +355,66 @@ private void update(final LRUCacheEntry entry) {
 
     private static class NamedCacheMetrics {
         private final StreamsMetricsImpl metrics;
-        private final String groupName;
-        private final Map metricTags;
-        private final Map allMetricTags;
+
         private final Sensor hitRatioSensor;
+        private final String taskName;
+        private final String cacheName;
 
-        private NamedCacheMetrics(final StreamsMetrics metrics, final String name) {
-            final String scope = "record-cache";
-            final String opName = "hitRatio";
-            final String tagKey = scope + "-id";
-            final String tagValue = ThreadCache.underlyingStoreNamefromCacheName(name);
-            this.groupName = "stream-" + scope + "-metrics";
-            this.metrics = (StreamsMetricsImpl) metrics;
-            this.allMetricTags = ((StreamsMetricsImpl) metrics).tagMap(tagKey, "all",
-                "task-id", ThreadCache.taskIDfromCacheName(name));
-            this.metricTags = ((StreamsMetricsImpl) metrics).tagMap(tagKey, tagValue,
-                "task-id", ThreadCache.taskIDfromCacheName(name));
+        private NamedCacheMetrics(final StreamsMetricsImpl metrics, final String cacheName) {
+            taskName = ThreadCache.taskIDfromCacheName(cacheName);
+            this.cacheName = cacheName;
+            this.metrics = metrics;
+            final String group = "stream-record-cache-metrics";
 
             // add parent
-            final Sensor parent = this.metrics.registry().sensor(opName, Sensor.RecordingLevel.DEBUG);
-            parent.add(this.metrics.registry().metricName(opName + "-avg", groupName,
-                    "The average cache hit ratio.", allMetricTags), new Avg());
-            parent.add(this.metrics.registry().metricName(opName + "-min", groupName,
-                    "The minimum cache hit ratio.", allMetricTags), new Min());
-            parent.add(this.metrics.registry().metricName(opName + "-max", groupName,
-                    "The maximum cache hit ratio.", allMetricTags), new Max());
+            final Map allMetricTags = metrics.tagMap(
+                "record-cache-id", "all",
+                "task-id", taskName
+            );
+            final Sensor taskLevelHitRatioSensor = metrics.taskLevelSensor("hitRatio", taskName, Sensor.RecordingLevel.DEBUG);
+            taskLevelHitRatioSensor.add(
+                new MetricName("hitRatio-avg", group, "The average cache hit ratio.", allMetricTags),
+                new Avg()
+            );
+            taskLevelHitRatioSensor.add(
+                new MetricName("hitRatio-min", group, "The minimum cache hit ratio.", allMetricTags),
+                new Min()
+            );
+            taskLevelHitRatioSensor.add(
+                new MetricName("hitRatio-max", group, "The maximum cache hit ratio.", allMetricTags),
+                new Max()
+            );
 
             // add child
-            hitRatioSensor = this.metrics.registry().sensor(opName, Sensor.RecordingLevel.DEBUG, parent);
-            hitRatioSensor.add(this.metrics.registry().metricName(opName + "-avg", groupName,
-                    "The average cache hit ratio.", metricTags), new Avg());
-            hitRatioSensor.add(this.metrics.registry().metricName(opName + "-min", groupName,
-                    "The minimum cache hit ratio.", metricTags), new Min());
-            hitRatioSensor.add(this.metrics.registry().metricName(opName + "-max", groupName,
-                    "The maximum cache hit ratio.", metricTags), new Max());
+            final Map metricTags = metrics.tagMap(
+                "record-cache-id", ThreadCache.underlyingStoreNamefromCacheName(cacheName),
+                "task-id", taskName
+            );
+
+            hitRatioSensor = metrics.cacheLevelSensor(
+                taskName,
+                cacheName,
+                "hitRatio",
+                Sensor.RecordingLevel.DEBUG,
+                taskLevelHitRatioSensor
+            );
+            hitRatioSensor.add(
+                new MetricName("hitRatio-avg", group, "The average cache hit ratio.", metricTags),
+                new Avg()
+            );
+            hitRatioSensor.add(
+                new MetricName("hitRatio-min", group, "The minimum cache hit ratio.", metricTags),
+                new Min()
+            );
+            hitRatioSensor.add(
+                new MetricName("hitRatio-max", group, "The maximum cache hit ratio.", metricTags),
+                new Max()
+            );
 
         }
 
         private void removeAllSensors() {
-            metrics.removeSensor(hitRatioSensor);
+            metrics.removeAllCacheLevelSensors(taskName, cacheName);
         }
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
index b1fd198916877..b947664fe2972 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
@@ -19,8 +19,8 @@
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.StreamsMetrics;
 import org.apache.kafka.streams.processor.internals.RecordContext;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.slf4j.Logger;
 
 import java.util.Collections;
@@ -39,7 +39,7 @@
 public class ThreadCache {
     private final Logger log;
     private final long maxCacheSizeBytes;
-    private final StreamsMetrics metrics;
+    private final StreamsMetricsImpl metrics;
     private final Map caches = new HashMap<>();
 
     // internal stats
@@ -52,7 +52,7 @@ public interface DirtyEntryFlushListener {
         void apply(final List dirty);
     }
 
-    public ThreadCache(final LogContext logContext, long maxCacheSizeBytes, final StreamsMetrics metrics) {
+    public ThreadCache(final LogContext logContext, long maxCacheSizeBytes, final StreamsMetricsImpl metrics) {
         this.maxCacheSizeBytes = maxCacheSizeBytes;
         this.metrics = metrics;
         this.log = logContext.logger(getClass());
@@ -91,7 +91,7 @@ public static String nameSpaceFromTaskIdAndStore(final String taskIDString, fina
      * @return
      */
     public static String taskIDfromCacheName(final String cacheName) {
-        String[] tokens = cacheName.split("-");
+        String[] tokens = cacheName.split("-", 2);
         return tokens[0];
     }
 
@@ -101,7 +101,7 @@ public static String taskIDfromCacheName(final String cacheName) {
      * @return
      */
     public static String underlyingStoreNamefromCacheName(final String cacheName) {
-        String[] tokens = cacheName.split("-");
+        String[] tokens = cacheName.split("-", 2);
         return tokens[1];
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
index 301d448ae65cb..8cb2eaebe1c7d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
@@ -121,7 +121,6 @@ private void initStore(final boolean enableCaching) {
 
     @After
     public void closeStore() {
-        context.close();
         sessionStore.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
index 1409d683e8952..a7a26107bf223 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java
@@ -148,7 +148,6 @@ public void testMetrics() {
             "The average number of occurrence of " + throughputOperation + " operation per second.", metricTags)));
 
 
-        context.close();
     }
 
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/SinkNodeTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/SinkNodeTest.java
index 0013167fb2463..753d26bd86e6b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/SinkNodeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/SinkNodeTest.java
@@ -26,7 +26,6 @@
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.state.StateSerdes;
 import org.apache.kafka.test.InternalMockProcessorContext;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -55,11 +54,6 @@ public void before() {
         sink.init(context);
     }
 
-    @After
-    public void after() {
-        context.close();
-    }
-
     @Test
     @SuppressWarnings("unchecked")
     public void shouldThrowStreamsExceptionOnInputRecordWithInvalidTimestamp() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index 28e0b46b2a62a..598e47ee13ebd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -232,7 +232,6 @@ public void testProcessOrder() {
     public void testMetrics() {
         task = createStatelessTask(createConfig(false));
 
-        assertNotNull(metrics.getSensor("commit"));
         assertNotNull(getMetric("%s-latency-avg", "The average latency of %s operation.", task.id().toString()));
         assertNotNull(getMetric("%s-latency-max", "The max latency of %s operation.", task.id().toString()));
         assertNotNull(getMetric("%s-rate", "The average number of occurrence of %s operation per second.", task.id().toString()));
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 36a1bce8cf0c4..3ae7acbc25d24 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -243,17 +243,8 @@ private StreamThread createStreamThread(@SuppressWarnings("SameParameterValue")
     public void testMetricsCreatedAtStartup() {
         final StreamThread thread = createStreamThread(clientId, config, false);
         final String defaultGroupName = "stream-metrics";
-        final String defaultPrefix = "thread." + thread.getName();
         final Map defaultTags = Collections.singletonMap("client-id", thread.getName());
 
-        assertNotNull(metrics.getSensor(defaultPrefix + ".commit-latency"));
-        assertNotNull(metrics.getSensor(defaultPrefix + ".poll-latency"));
-        assertNotNull(metrics.getSensor(defaultPrefix + ".process-latency"));
-        assertNotNull(metrics.getSensor(defaultPrefix + ".punctuate-latency"));
-        assertNotNull(metrics.getSensor(defaultPrefix + ".task-created"));
-        assertNotNull(metrics.getSensor(defaultPrefix + ".task-closed"));
-        assertNotNull(metrics.getSensor(defaultPrefix + ".skipped-records"));
-
         assertNotNull(metrics.metrics().get(metrics.metricName("commit-latency-avg", defaultGroupName, "The average commit time in ms", defaultTags)));
         assertNotNull(metrics.metrics().get(metrics.metricName("commit-latency-max", defaultGroupName, "The maximum commit time in ms", defaultTags)));
         assertNotNull(metrics.metrics().get(metrics.metricName("commit-rate", defaultGroupName, "The average per-second number of commit calls", defaultTags)));
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
index 51c782ad9d506..c4536dfd2a224 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
@@ -63,7 +63,6 @@ public void before() {
     @After
     public void after() {
         store.close();
-        context.close();
         driver.clear();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java
index 8705326b78210..3e0241e3043c4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingKeyValueStoreTest.java
@@ -82,7 +82,6 @@ public void setUp() {
     @After
     public void after() {
         super.after();
-        context.close();
     }
 
     @SuppressWarnings("unchecked")
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java
index a9a66e9a7df89..b77f4e979d5ba 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java
@@ -82,7 +82,6 @@ public void setUp() {
 
     @After
     public void close() {
-        context.close();
         cachingStore.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
index c25655b6b2ea1..a87b2e4189024 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
@@ -87,7 +87,6 @@ public void setUp() {
 
     @After
     public void closeStore() {
-        context.close();
         cachingStore.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStoreTest.java
index 7342c93abccbe..5bb0de7a0ebc3 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingKeyValueBytesStoreTest.java
@@ -76,7 +76,6 @@ public  void send(final String topic,
 
     @After
     public void after() {
-        context.close();
         store.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
index eab523e398d04..19bd523a8d54e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java
@@ -35,7 +35,6 @@
 import org.apache.kafka.test.StreamsTestUtils;
 import org.apache.kafka.test.TestUtils;
 import org.easymock.EasyMock;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -83,11 +82,6 @@ public RecordCollector recordCollector() {
         );
     }
 
-    @After
-    public void after() {
-        context.close();
-    }
-
     @Test
     public void shouldRecordRestoreLatencyOnInit() {
         innerStoreMock.init(context, store);
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
index 6b410dc73ddb2..9ae0feb1850b1 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
@@ -16,12 +16,11 @@
  */
 package org.apache.kafka.streams.state.internals;
 
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.metrics.KafkaMetric;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.processor.internals.MockStreamsMetrics;
+import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -44,13 +43,14 @@
 public class NamedCacheTest {
 
     private NamedCache cache;
-    private MockStreamsMetrics streamMetrics;
+    private StreamsMetricsImpl metrics;
     private final String taskIDString = "0.0";
     private final String underlyingStoreName = "storeName";
+
     @Before
     public void setUp() {
-        streamMetrics = new MockStreamsMetrics(new Metrics());
-        cache = new NamedCache(taskIDString + "-" + underlyingStoreName, streamMetrics);
+        metrics = new MockStreamsMetrics(new Metrics());
+        cache = new NamedCache(taskIDString + "-" + underlyingStoreName, metrics);
     }
 
     @Test
@@ -83,18 +83,15 @@ public void testMetrics() {
         metricTags.put("task-id", taskIDString);
         metricTags.put("client-id", "test");
 
-        assertNotNull(streamMetrics.registry().getSensor("hitRatio"));
-        final Map metrics1 = streamMetrics.registry().metrics();
-        getMetricByNameFilterByTags(metrics1, "hitRatio-avg", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics1, "hitRatio-min", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics1, "hitRatio-max", "stream-record-cache-metrics", metricTags);
+        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-avg", "stream-record-cache-metrics", metricTags);
+        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-min", "stream-record-cache-metrics", metricTags);
+        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-max", "stream-record-cache-metrics", metricTags);
 
         // test "all"
         metricTags.put("record-cache-id", "all");
-        final Map metrics = streamMetrics.registry().metrics();
-        getMetricByNameFilterByTags(metrics, "hitRatio-avg", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics, "hitRatio-min", "stream-record-cache-metrics", metricTags);
-        getMetricByNameFilterByTags(metrics, "hitRatio-max", "stream-record-cache-metrics", metricTags);
+        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-avg", "stream-record-cache-metrics", metricTags);
+        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-min", "stream-record-cache-metrics", metricTags);
+        getMetricByNameFilterByTags(metrics.metrics(), "hitRatio-max", "stream-record-cache-metrics", metricTags);
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplierTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplierTest.java
index 098c3262e3456..b25b8cbb00a66 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplierTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplierTest.java
@@ -53,7 +53,6 @@ public class RocksDBKeyValueStoreSupplierTest {
 
     @After
     public void close() {
-        context.close();
         store.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java
index 388a2fc47ba10..bd2fa9110a5ba 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java
@@ -82,7 +82,6 @@ public void before() {
 
     @After
     public void close() {
-        context.close();
         bytesStore.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplierTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplierTest.java
index 272e0b0f5e2fc..c50dfba2942d6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplierTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplierTest.java
@@ -68,7 +68,6 @@ public  void send(final String topic,
 
     @After
     public void close() {
-        context.close();
         store.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java
index 64953153045d9..bcb411b0709eb 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java
@@ -69,7 +69,6 @@ public void before() {
 
     @After
     public void close() {
-        context.close();
         sessionStore.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
index a09d87dbad3eb..b7a9d375d9c43 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBStoreTest.java
@@ -79,7 +79,6 @@ public void setUp() {
     @After
     public void tearDown() {
         rocksDBStore.close();
-        context.close();
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplierTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplierTest.java
index a6ccfdf9337cd..7409a13c1544c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplierTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplierTest.java
@@ -53,7 +53,6 @@ public class RocksDBWindowStoreSupplierTest {
 
     @After
     public void close() {
-        context.close();
         if (store != null) {
             store.close();
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
index bf556adebfa21..92edbd8d31c3f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
@@ -126,7 +126,6 @@ private WindowStore createWindowStore(final ProcessorContext co
 
     @After
     public void closeStore() {
-        context.close();
         if (windowStore != null) {
             windowStore.close();
         }
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
index d61218eb9b500..7a7b266537b4f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
@@ -76,7 +76,6 @@ public void closeSegments() {
         }
         segmentOne.close();
         segmentTwo.close();
-        context.close();
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
index ec59a008112fe..bfa317ddc5197 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
@@ -65,7 +65,6 @@ public void createContext() {
 
     @After
     public void close() {
-        context.close();
         segments.close();
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
index 21b5c5c06cdb2..6bacd910ccece 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
@@ -25,7 +25,6 @@
 import org.apache.kafka.streams.processor.internals.RecordCollectorImpl;
 import org.apache.kafka.streams.state.StateSerdes;
 import org.apache.kafka.test.InternalMockProcessorContext;
-import org.junit.After;
 import org.junit.Test;
 
 import java.util.HashMap;
@@ -68,11 +67,6 @@ public  void send(final String topic,
 
     private final StoreChangeLogger changeLogger = new StoreChangeLogger<>(topic, context, StateSerdes.withBuiltinTypes(topic, Integer.class, String.class));
 
-    @After
-    public void after() {
-        context.close();
-    }
-
     @Test
     public void testAddRemove() {
         context.setTime(1);
diff --git a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
index 57e3efb1b1631..5e619102ede60 100644
--- a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
+++ b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java
@@ -51,7 +51,6 @@
 public class InternalMockProcessorContext extends AbstractProcessorContext implements RecordCollector.Supplier {
 
     private final File stateDir;
-    private final Metrics metrics;
     private final RecordCollector.Supplier recordCollectorSupplier;
     private final Map storeMap = new LinkedHashMap<>();
     private final Map restoreFuncs = new HashMap<>();
@@ -135,7 +134,6 @@ public InternalMockProcessorContext(final File stateDir,
         this.stateDir = stateDir;
         this.keySerde = keySerde;
         this.valSerde = valSerde;
-        this.metrics = metrics.registry();
         this.recordCollectorSupplier = collectorSupplier;
     }
 
@@ -306,10 +304,6 @@ public void restore(final String storeName, final Iterable stores) {
         for (final StateStore store : stores) {
-            store.init(context, store);
+            try {
+                store.init(context, store);
+            } catch (final RuntimeException e) {
+                new RuntimeException("Fatal exception initializing store.", e).printStackTrace();
+                throw e;
+            }
         }
 
         for (final ProcessorNode node : topology.processors()) {
@@ -230,7 +235,6 @@ public void close() {
         }
 
         closeState();
-        context.close();
     }
 
     public Set allProcessorNames() {

From e7f019690a10306a717b9757a21d5bf01964532e Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Thu, 26 Apr 2018 13:04:59 -0400
Subject: [PATCH 0297/1847] MINOR: Fixes for streams system tests (#4935)

This PR fixes some regressions introduced into streams system tests and sets the upgrade tests to ignore until PR #4636 is merged as it has the fixes for the upgrade tests.

Reviewers: Guozhang Wang 
---
 tests/kafkatest/services/streams.py                   | 8 +++-----
 tests/kafkatest/tests/streams/streams_upgrade_test.py | 9 ++++++---
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py
index a4b902a322c45..ec6a08157f1d1 100644
--- a/tests/kafkatest/services/streams.py
+++ b/tests/kafkatest/services/streams.py
@@ -15,16 +15,15 @@
 
 import os.path
 import signal
-
 import streams_property
-
 from ducktape.services.service import Service
 from ducktape.utils.util import wait_until
 from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
-from kafkatest.services.monitor.jmx import JmxMixin
 from kafkatest.services.kafka import KafkaConfig
+from kafkatest.services.monitor.jmx import JmxMixin
 from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1
 
+STATE_DIR = "state.dir"
 
 class StreamsTestBaseService(KafkaPathResolverMixin, JmxMixin, Service):
     """Base class for Streams Test services providing some common settings and functionality"""
@@ -308,7 +307,6 @@ def disable_auto_terminate(self):
 
     def start_cmd(self, node):
         args = self.args.copy()
-        args['kafka'] = self.kafka.bootstrap_servers()
         args['config_file'] = self.CONFIG_FILE
         args['stdout'] = self.STDOUT_FILE
         args['stderr'] = self.STDERR_FILE
@@ -319,7 +317,7 @@ def start_cmd(self, node):
 
         cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \
               "INCLUDE_TEST_JARS=true %(kafka_run_class)s %(streams_class_name)s " \
-              " %(kafka)s %(config_file)s %(user_test_args)s %(disable_auto_terminate)s" \
+              " %(config_file)s %(user_test_args1)s %(disable_auto_terminate)s" \
               " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args
 
         return cmd
diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py
index 8b7d7712459a1..debe85fd7e271 100644
--- a/tests/kafkatest/tests/streams/streams_upgrade_test.py
+++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py
@@ -13,15 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ducktape.mark import ignore, matrix, parametrize
+import random
+import time
+from ducktape.mark import ignore, matrix
 from ducktape.mark.resource import cluster
 from ducktape.tests.test import Test
 from kafkatest.services.kafka import KafkaService
 from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService, StreamsUpgradeTestJobRunnerService
 from kafkatest.services.zookeeper import ZookeeperService
 from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, DEV_BRANCH, DEV_VERSION, KafkaVersion
-import random
-import time
 
 # broker 0.10.0 is not compatible with newer Kafka Streams versions
 broker_upgrade_versions = [str(LATEST_0_10_1), str(LATEST_0_10_2), str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1), str(DEV_BRANCH)]
@@ -56,6 +56,7 @@ def perform_broker_upgrade(self, to_version):
             node.version = KafkaVersion(to_version)
             self.kafka.start_node(node)
 
+    @ignore
     @cluster(num_nodes=6)
     @matrix(from_version=broker_upgrade_versions, to_version=broker_upgrade_versions)
     def test_upgrade_downgrade_brokers(self, from_version, to_version):
@@ -123,6 +124,7 @@ def test_upgrade_downgrade_brokers(self, from_version, to_version):
         node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False)
         self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False)
 
+    @ignore
     @matrix(from_version=metadata_2_versions, to_version=metadata_2_versions)
     def test_simple_upgrade_downgrade(self, from_version, to_version):
         """
@@ -175,6 +177,7 @@ def test_simple_upgrade_downgrade(self, from_version, to_version):
         self.driver.stop()
 
     #@matrix(from_version=metadata_1_versions, to_version=backward_compatible_metadata_2_versions)
+    @ignore
     @matrix(from_version=metadata_1_versions, to_version=metadata_3_versions)
     @matrix(from_version=metadata_2_versions, to_version=metadata_3_versions)
     def test_metadata_upgrade(self, from_version, to_version):

From f38bc0c33992ed4879ec37e0d425beadc088bbff Mon Sep 17 00:00:00 2001
From: Jarek Rudzinski 
Date: Fri, 27 Apr 2018 02:50:22 +0900
Subject: [PATCH 0298/1847] MINOR: use jdk8 to build/run system tests (#4925)

Debian installer packages are no longer available for Java 7.

Also upgrade AMI to latest ubuntu/trusty 14 amd64 as the older
one is no longer available.

Note that this only changes the JDK used to build and run
the system tests. We still have Jenkins jobs that compile
and run the JUnit tests with Java 7 so that we don't use
features that are only available in newer Java versions.
---
 Vagrantfile     |  4 ++--
 vagrant/base.sh | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 069bc5be8861a..363607691fd0d 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -40,7 +40,7 @@ ec2_keypair_file = nil
 
 ec2_region = "us-east-1"
 ec2_az = nil # Uses set by AWS
-ec2_ami = "ami-9eaa1cf6"
+ec2_ami = "ami-905730e8"
 ec2_instance_type = "m3.medium"
 ec2_user = "ubuntu"
 ec2_instance_name_prefix = "kafka-vagrant"
@@ -80,7 +80,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       # share to a temporary location and the provisioning scripts symlink data
       # to the right location.
       override.cache.enable :generic, {
-        "oracle-jdk7" => { cache_dir: "/tmp/oracle-jdk7-installer-cache" },
+        "oracle-jdk8" => { cache_dir: "/tmp/oracle-jdk8-installer-cache" },
       }
     end
   end
diff --git a/vagrant/base.sh b/vagrant/base.sh
index f5c03cca4fdf2..33ee056cf438a 100755
--- a/vagrant/base.sh
+++ b/vagrant/base.sh
@@ -27,13 +27,13 @@ if [ -z `which javac` ]; then
     apt-get -y update
 
     # Try to share cache. See Vagrantfile for details
-    mkdir -p /var/cache/oracle-jdk7-installer
-    if [ -e "/tmp/oracle-jdk7-installer-cache/" ]; then
-        find /tmp/oracle-jdk7-installer-cache/ -not -empty -exec cp '{}' /var/cache/oracle-jdk7-installer/ \;
+    mkdir -p /var/cache/oracle-jdk8-installer
+    if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then
+        find /tmp/oracle-jdk8-installer-cache/ -not -empty -exec cp '{}' /var/cache/oracle-jdk8-installer/ \;
     fi
-    if [ ! -e "/var/cache/oracle-jdk7-installer/jdk-7u80-linux-x64.tar.gz" ]; then
-        # Grab a copy of the JDK since it has moved and original downloader won't work
-        curl -s -L "https://s3-us-west-2.amazonaws.com/kafka-packages/jdk-7u80-linux-x64.tar.gz" -o /var/cache/oracle-jdk7-installer/jdk-7u80-linux-x64.tar.gz
+    if [ ! -e "/var/cache/oracle-jdk8-installer/jdk-8u171-linux-x64.tar.gz" ]; then
+      # Grab a copy of the JDK since it has moved and original downloader won't work
+      curl -s -L "https://s3-us-west-2.amazonaws.com/kafka-packages/jdk-8u171-linux-x64.tar.gz" -o /var/cache/oracle-jdk8-installer/jdk-8u171-linux-x64.tar.gz
     fi
 
     /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
@@ -42,15 +42,15 @@ if [ -z `which javac` ]; then
     # as one line per dot in the build logs.
     # To avoid this noise we redirect all output to a file that we only show if apt-get fails.
     echo "Installing JDK..."
-    if ! apt-get -y install oracle-java7-installer oracle-java7-set-default >/tmp/jdk_install.log 2>&1 ; then
+    if ! apt-get -y install oracle-java8-installer oracle-java8-set-default >/tmp/jdk_install.log 2>&1 ; then
         cat /tmp/jdk_install.log
         echo "ERROR: JDK install failed"
         exit 1
     fi
     echo "JDK installed: $(javac -version 2>&1)"
 
-    if [ -e "/tmp/oracle-jdk7-installer-cache/" ]; then
-        cp -R /var/cache/oracle-jdk7-installer/* /tmp/oracle-jdk7-installer-cache
+    if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then
+        cp -R /var/cache/oracle-jdk8-installer/* /tmp/oracle-jdk8-installer-cache
     fi
 fi
 

From 885abbfcd40aab57acec278d976956f07be15090 Mon Sep 17 00:00:00 2001
From: Filipe Agapito 
Date: Thu, 26 Apr 2018 19:30:42 +0100
Subject: [PATCH 0299/1847] KAFKA-6474: Rewrite tests to use new public
 TopologyTestDriver [partial] (#4832)

* Remove ProcessorTopologyTestDriver from TopologyTest
* Fix ProcessorTopologyTest
* Remove ProcessorTopologyTestDriver and InternalTopologyAccessor
* Partially refactored StreamsBuilderTest but missing one test
* Refactor KStreamBuilderTest
* Refactor AbstractStreamTest
* Further cleanup of AbstractStreamTest
* Refactor GlobalKTableJoinsTest
* Refactor InternalStreamsBuilderTest
* Fix circular dependency in build.gradle
* Refactor KGroupedStreamImplTest
* Partial modifications to KGroupedTableImplTest
* Refactor KGroupedTableImplTest
* Refactor KStreamBranchTest
* Refactor KStreamFilterTest
* Refactor KStreamFlatMapTest KStreamFlatMapValuesTest
* Refactor KStreamForeachTest
* Refactor KStreamGlobalKTableJoinTest
* Refactor KStreamGlobalKTableLeftJoinTest
* Refactor KStreamImplTest
* Refactor KStreamImplTest
* Refactor KStreamKStreamJoinTest
* Refactor KStreamKStreamLeftJoinTest
* Refactor KStreamKTableJoinTest
* Refactor KStreamKTableLeftJoinTest
* Refactor KStreamMapTest and KStreamMapValuesTest
* Refactor KStreamPeekTest and KStreamTransformTest
* Refactor KStreamSelectKeyTest
* Refactor KStreamTransformValuesTest
* Refactor KStreamWindowAggregateTest
* Add Depercation anotation to KStreamTestDriver and rollback failing tests in StreamsBuilderTest and KTableAggregateTest
* Refactor KTableFilterTest
* Refactor KTableForeachTest
* Add getter for ProcessorTopology, and simplify tests in StreamsBuilderTest
* Refactor KTableImplTest
* Remove unused imports
* Refactor KTableAggregateTest
* Fix style errors
* Fix gradle build
* Address reviewer comments:
  - Remove properties new instance
  - Remove extraneous line
  - Remove unnecessary TopologyTestDriver instances from StreamsBuilderTest
  - Move props.clear() to @After
  - Clarify use of timestamp in KStreamFlatMapValuesTest
  - Keep test using old Punctuator in KStreamTransformTest
  - Add comment to clarify clock advances in KStreamTransformTest
  - Add TopologyTestDriverWrapper class to access the protected constructor of TopologyTestDriver
  - Revert KTableImplTest.testRepartition to KStreamTestDriver to avoid exposing the TopologyTestDriver processor topology
  - Revert partially migrated classes: KTableAggregateTest, KTableFilterTest, and KTableImplTest
* Rebase on current trunk an fix conflicts

Reviewers: Matthias J Sax , Bill Bejeck , John Roesler 
---
 .../kafka/streams/StreamsBuilderTest.java     | 113 ++--
 .../apache/kafka/streams/TopologyTest.java    |   4 +-
 ...or.java => TopologyTestDriverWrapper.java} |  17 +-
 .../streams/kstream/KStreamBuilderTest.java   |  53 +-
 .../kstream/internals/AbstractStreamTest.java |  23 +-
 .../internals/GlobalKTableJoinsTest.java      |  45 +-
 .../internals/InternalStreamsBuilderTest.java |  38 +-
 .../internals/KGroupedStreamImplTest.java     | 143 +++--
 .../internals/KGroupedTableImplTest.java      |  67 ++-
 .../kstream/internals/KStreamBranchTest.java  |  39 +-
 .../kstream/internals/KStreamFilterTest.java  |  46 +-
 .../kstream/internals/KStreamFlatMapTest.java |  37 +-
 .../internals/KStreamFlatMapValuesTest.java   |  42 +-
 .../kstream/internals/KStreamForeachTest.java |  44 +-
 .../KStreamGlobalKTableJoinTest.java          |  51 +-
 .../KStreamGlobalKTableLeftJoinTest.java      |  34 +-
 .../kstream/internals/KStreamImplTest.java    |  93 ++--
 .../internals/KStreamKStreamJoinTest.java     | 354 ++++++-------
 .../internals/KStreamKStreamLeftJoinTest.java | 122 ++---
 .../internals/KStreamKTableJoinTest.java      |  52 +-
 .../internals/KStreamKTableLeftJoinTest.java  |  35 +-
 .../kstream/internals/KStreamMapTest.java     |  39 +-
 .../internals/KStreamMapValuesTest.java       |  43 +-
 .../kstream/internals/KStreamPeekTest.java    |  39 +-
 .../internals/KStreamSelectKeyTest.java       |  38 +-
 .../internals/KStreamTransformTest.java       | 107 +++-
 .../internals/KStreamTransformValuesTest.java |  44 +-
 .../internals/KStreamWindowAggregateTest.java | 200 +++----
 .../kstream/internals/KTableForeachTest.java  |  39 +-
 .../processor/TopologyBuilderTest.java        |  17 +-
 .../InternalTopologyBuilderTest.java          |   4 +-
 .../internals/ProcessorTopologyTest.java      | 115 ++--
 .../apache/kafka/test/KStreamTestDriver.java  |   1 +
 .../test/ProcessorTopologyTestDriver.java     | 490 ------------------
 .../kafka/streams/TopologyTestDriver.java     |  28 +-
 35 files changed, 1290 insertions(+), 1366 deletions(-)
 rename streams/src/test/java/org/apache/kafka/streams/{InternalTopologyAccessor.java => TopologyTestDriverWrapper.java} (58%)
 delete mode 100644 streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java

diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
index 4a496b8dd282d..d3e01faf32ba2 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java
@@ -16,7 +16,9 @@
  */
 package org.apache.kafka.streams;
 
+import org.apache.kafka.common.serialization.LongSerializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.errors.TopologyException;
@@ -28,13 +30,14 @@
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockPredicate;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
-import org.junit.Rule;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -43,6 +46,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -54,9 +58,26 @@
 public class StreamsBuilderTest {
 
     private final StreamsBuilder builder = new StreamsBuilder();
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "streams-builder-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
 
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test(expected = TopologyException.class)
     public void testFrom() {
@@ -70,9 +91,7 @@ public void shouldAllowJoinUnmaterializedFilteredKTable() {
         final KTable filteredKTable = builder.table("table-topic").filter(MockPredicate.allGoodPredicate());
         builder.stream("stream-topic").join(filteredKTable, MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(1));
         assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000005"), equalTo(Collections.singleton(topology.stateStores().get(0).name())));
@@ -85,9 +104,7 @@ public void shouldAllowJoinMaterializedFilteredKTable() {
                 .filter(MockPredicate.allGoodPredicate(), Materialized.>as("store"));
         builder.stream("stream-topic").join(filteredKTable, MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(2));
         assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000005"), equalTo(Collections.singleton("store")));
@@ -99,9 +116,7 @@ public void shouldAllowJoinUnmaterializedMapValuedKTable() {
         final KTable mappedKTable = builder.table("table-topic").mapValues(MockMapper.noOpValueMapper());
         builder.stream("stream-topic").join(mappedKTable, MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(1));
         assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000005"), equalTo(Collections.singleton(topology.stateStores().get(0).name())));
@@ -114,9 +129,7 @@ public void shouldAllowJoinMaterializedMapValuedKTable() {
                 .mapValues(MockMapper.noOpValueMapper(), Materialized.>as("store"));
         builder.stream("stream-topic").join(mappedKTable, MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(2));
         assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000005"), equalTo(Collections.singleton("store")));
@@ -129,14 +142,11 @@ public void shouldAllowJoinUnmaterializedJoinedKTable() {
         final KTable table2 = builder.table("table-topic2");
         builder.stream("stream-topic").join(table1.join(table2, MockValueJoiner.TOSTRING_JOINER), MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(2));
         assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000010"), equalTo(Utils.mkSet(topology.stateStores().get(0).name(), topology.stateStores().get(1).name())));
         assertThat(topology.processorConnectedStateStores("KTABLE-MERGE-0000000007").isEmpty(), is(true));
-
     }
 
     @Test
@@ -145,9 +155,7 @@ public void shouldAllowJoinMaterializedJoinedKTable() {
         final KTable table2 = builder.table("table-topic2");
         builder.stream("stream-topic").join(table1.join(table2, MockValueJoiner.TOSTRING_JOINER, Materialized.>as("store")), MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(3));
         assertThat(topology.processorConnectedStateStores("KSTREAM-JOIN-0000000010"), equalTo(Collections.singleton("store")));
@@ -159,9 +167,7 @@ public void shouldAllowJoinMaterializedSourceKTable() {
         final KTable table = builder.table("table-topic");
         builder.stream("stream-topic").join(table, MockValueJoiner.TOSTRING_JOINER);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-
-        ProcessorTopology topology = builder.internalTopologyBuilder.build();
+        final ProcessorTopology topology = builder.internalTopologyBuilder.build();
 
         assertThat(topology.stateStores().size(), equalTo(1));
         assertThat(topology.processorConnectedStateStores("KTABLE-SOURCE-0000000002"), equalTo(Collections.singleton(topology.stateStores().get(0).name())));
@@ -177,10 +183,10 @@ public void shouldProcessingFromSinkTopic() {
 
         source.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process("topic-source", "A", "aa");
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+        driver.pipeInput(recordFactory.create("topic-source", "A", "aa"));
 
         // no exception was thrown
         assertEquals(Utils.mkList("A:aa"), processorSupplier.processed);
@@ -197,10 +203,10 @@ public void shouldProcessViaThroughTopic() {
         source.process(sourceProcessorSupplier);
         through.process(throughProcessorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process("topic-source", "A", "aa");
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+        driver.pipeInput(recordFactory.create("topic-source", "A", "aa"));
 
         assertEquals(Utils.mkList("A:aa"), sourceProcessorSupplier.processed);
         assertEquals(Utils.mkList("A:aa"), throughProcessorSupplier.processed);
@@ -218,13 +224,13 @@ public void testMerge() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         merged.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process(topic1, "A", "aa");
-        driver.process(topic2, "B", "bb");
-        driver.process(topic2, "C", "cc");
-        driver.process(topic1, "D", "dd");
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+        driver.pipeInput(recordFactory.create(topic1, "A", "aa"));
+        driver.pipeInput(recordFactory.create(topic2, "B", "bb"));
+        driver.pipeInput(recordFactory.create(topic2, "C", "cc"));
+        driver.pipeInput(recordFactory.create(topic1, "D", "dd"));
 
         assertEquals(Utils.mkList("A:aa", "B:bb", "C:cc", "D:dd"), processorSupplier.processed);
     }
@@ -244,12 +250,13 @@ public void apply(final Long key, final String value) {
                 .withValueSerde(Serdes.String()))
                 .toStream().foreach(action);
 
-        driver.setUp(builder, TestUtils.tempDirectory());
-        driver.setTime(0L);
-        driver.process(topic, 1L, "value1");
-        driver.process(topic, 2L, "value2");
-        driver.flushState();
-        final KeyValueStore store = (KeyValueStore) driver.allStateStores().get("store");
+        driver = new TopologyTestDriver(builder.build(), props);
+
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new LongSerializer(), new StringSerializer());
+        driver.pipeInput(recordFactory.create(topic, 1L, "value1"));
+        driver.pipeInput(recordFactory.create(topic, 2L, "value2"));
+
+        final KeyValueStore store = driver.getKeyValueStore("store");
         assertThat(store.get(1L), equalTo("value1"));
         assertThat(store.get(2L), equalTo("value2"));
         assertThat(results.get(1L), equalTo("value1"));
@@ -262,12 +269,14 @@ public void shouldUseSerdesDefinedInMaterializedToConsumeGlobalTable() {
         builder.globalTable(topic, Materialized.>as("store")
                 .withKeySerde(Serdes.Long())
                 .withValueSerde(Serdes.String()));
-        driver.setUp(builder, TestUtils.tempDirectory());
-        driver.setTime(0L);
-        driver.process(topic, 1L, "value1");
-        driver.process(topic, 2L, "value2");
-        driver.flushState();
-        final KeyValueStore store = (KeyValueStore) driver.allStateStores().get("store");
+
+        driver = new TopologyTestDriver(builder.build(), props);
+
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new LongSerializer(), new StringSerializer());
+        driver.pipeInput(recordFactory.create(topic, 1L, "value1"));
+        driver.pipeInput(recordFactory.create(topic, 2L, "value2"));
+        final KeyValueStore store = driver.getKeyValueStore("store");
+
         assertThat(store.get(1L), equalTo("value1"));
         assertThat(store.get(2L), equalTo("value2"));
     }
@@ -295,12 +304,12 @@ public void shouldUseDefaultNodeAndStoreNames() {
     }
     
     @Test(expected = TopologyException.class)
-    public void shouldThrowExceptionWhenNoTopicPresent() throws Exception {
+    public void shouldThrowExceptionWhenNoTopicPresent() {
         builder.stream(Collections.emptyList());
     }
 
     @Test(expected = NullPointerException.class)
-    public void shouldThrowExceptionWhenTopicNamesAreNull() throws Exception {
+    public void shouldThrowExceptionWhenTopicNamesAreNull() {
         builder.stream(Arrays.asList(null, null));
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java
index 68340910c2b83..eee3386053274 100644
--- a/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java
@@ -26,7 +26,6 @@
 import org.apache.kafka.streams.state.internals.KeyValueStoreBuilder;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockStateStore;
-import org.apache.kafka.test.ProcessorTopologyTestDriver;
 import org.apache.kafka.test.TestUtils;
 import org.easymock.EasyMock;
 import org.junit.Test;
@@ -261,7 +260,6 @@ public void shouldThrowOnUnassignedStateStoreAccess() throws Exception {
         config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "host:1");
         config.put(StreamsConfig.APPLICATION_ID_CONFIG, "appId");
         config.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
-        final StreamsConfig streamsConfig = new StreamsConfig(config);
         mockStoreBuilder();
         EasyMock.expect(storeBuilder.build()).andReturn(new MockStateStore("store", false));
         EasyMock.replay(storeBuilder);
@@ -274,7 +272,7 @@ public void shouldThrowOnUnassignedStateStoreAccess() throws Exception {
             .addProcessor(badNodeName, new LocalMockProcessorSupplier(), sourceNodeName);
 
         try {
-            new ProcessorTopologyTestDriver(streamsConfig, topology.internalTopologyBuilder);
+            new TopologyTestDriver(topology, config);
             fail("Should have thrown StreamsException");
         } catch (final StreamsException e) {
             final String error = e.toString();
diff --git a/streams/src/test/java/org/apache/kafka/streams/InternalTopologyAccessor.java b/streams/src/test/java/org/apache/kafka/streams/TopologyTestDriverWrapper.java
similarity index 58%
rename from streams/src/test/java/org/apache/kafka/streams/InternalTopologyAccessor.java
rename to streams/src/test/java/org/apache/kafka/streams/TopologyTestDriverWrapper.java
index c3c45046cbd76..fa976a80d687f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/InternalTopologyAccessor.java
+++ b/streams/src/test/java/org/apache/kafka/streams/TopologyTestDriverWrapper.java
@@ -14,19 +14,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.kafka.streams;
 
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 
+import java.util.Properties;
 
 /**
- * This class is meant for testing purposes only and allows the testing of
- * topologies by using the  {@link org.apache.kafka.test.ProcessorTopologyTestDriver}
+ *  This class allows the instantiation of a {@link TopologyTestDriver} using a
+ *  {@link InternalTopologyBuilder} by exposing a protected constructor.
+ *
+ *  It should be used only for testing, and should be removed once the deprecated
+ *  classes {@link org.apache.kafka.streams.kstream.KStreamBuilder} and
+ *  {@link org.apache.kafka.streams.processor.TopologyBuilder} are removed.
  */
-public class InternalTopologyAccessor {
+public class TopologyTestDriverWrapper extends TopologyTestDriver {
 
-    public static InternalTopologyBuilder getInternalTopologyBuilder(final Topology topology) {
-        return topology.internalTopologyBuilder;
+    public TopologyTestDriverWrapper(final InternalTopologyBuilder builder,
+                                     final Properties config) {
+        super(builder, config);
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java
index f9949d3c5b86e..81bdb31494826 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java
@@ -18,7 +18,10 @@
 
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriverWrapper;
 import org.apache.kafka.streams.errors.TopologyBuilderException;
 import org.apache.kafka.streams.kstream.internals.KStreamImpl;
 import org.apache.kafka.streams.kstream.internals.KTableImpl;
@@ -26,19 +29,21 @@
 import org.apache.kafka.streams.processor.TopologyBuilder;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.SourceNode;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockTimestampExtractor;
 import org.apache.kafka.test.MockValueJoiner;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -55,12 +60,27 @@ public class KStreamBuilderTest {
     private static final String APP_ID = "app-id";
 
     private final KStreamBuilder builder = new KStreamBuilder();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+    private TopologyTestDriverWrapper driver;
+    private final Properties props = new Properties();
 
     @Before
-    public void setUp() {
+    public void setup() {
         builder.setApplicationId(APP_ID);
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-builder-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test(expected = TopologyBuilderException.class)
@@ -93,10 +113,8 @@ public void shouldProcessFromSinkTopic() {
 
         source.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
-
-        driver.process("topic-source", "A", "aa");
+        driver = new TopologyTestDriverWrapper(builder.internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create("topic-source", "A", "aa"));
 
         // no exception was thrown
         assertEquals(Utils.mkList("A:aa"), processorSupplier.processed);
@@ -113,10 +131,8 @@ public void shouldProcessViaThroughTopic() {
         source.process(sourceProcessorSupplier);
         through.process(throughProcessorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
-
-        driver.process("topic-source", "A", "aa");
+        driver = new TopologyTestDriverWrapper(builder.internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create("topic-source", "A", "aa"));
 
         assertEquals(Utils.mkList("A:aa"), sourceProcessorSupplier.processed);
         assertEquals(Utils.mkList("A:aa"), throughProcessorSupplier.processed);
@@ -147,13 +163,12 @@ public void testMerge() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         merged.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriverWrapper(builder.internalTopologyBuilder, props);
 
-        driver.process(topic1, "A", "aa");
-        driver.process(topic2, "B", "bb");
-        driver.process(topic2, "C", "cc");
-        driver.process(topic1, "D", "dd");
+        driver.pipeInput(recordFactory.create(topic1, "A", "aa"));
+        driver.pipeInput(recordFactory.create(topic2, "B", "bb"));
+        driver.pipeInput(recordFactory.create(topic2, "C", "cc"));
+        driver.pipeInput(recordFactory.create(topic1, "D", "dd"));
 
         assertEquals(Utils.mkList("A:aa", "B:bb", "C:cc", "D:dd"), processorSupplier.processed);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java
index dcfb9ba5d789e..2aa07f356a041 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java
@@ -16,20 +16,25 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueTransformerSupplier;
 import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
+import java.util.Properties;
 import java.util.Random;
 
 import static org.easymock.EasyMock.createMock;
@@ -40,10 +45,6 @@
 
 public class AbstractStreamTest {
 
-    private final String topicName = "topic";
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
-
     @Test
     public void testToInternlValueTransformerSupplierSuppliesNewTransformers() {
         final ValueTransformerSupplier vts = createMock(ValueTransformerSupplier.class);
@@ -82,9 +83,15 @@ public void testShouldBeExtensible() {
 
         stream.randomFilter().process(processor);
 
-        driver.setUp(builder);
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "abstract-stream-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+        final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, "V" + expectedKey);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
         }
 
         assertTrue(processor.processed.size() <= expectedKeys.length);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java
index eebb7d2322649..8c50afeda6f44 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/GlobalKTableJoinsTest.java
@@ -17,22 +17,25 @@
 package org.apache.kafka.streams.kstream.internals;
 
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.GlobalKTable;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
@@ -43,17 +46,15 @@ public class GlobalKTableJoinsTest {
     private final Map results = new HashMap<>();
     private final String streamTopic = "stream";
     private final String globalTopic = "global";
-    private File stateDir;
     private GlobalKTable global;
     private KStream stream;
     private KeyValueMapper keyValueMapper;
     private ForeachAction action;
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private TopologyTestDriver driver;
+
 
     @Before
     public void setUp() {
-        stateDir = TestUtils.tempDirectory();
         final Consumed consumed = Consumed.with(Serdes.String(), Serdes.String());
         global = builder.globalTable(globalTopic, consumed);
         stream = builder.stream(streamTopic, consumed);
@@ -71,6 +72,14 @@ public void apply(final String key, final String value) {
         };
     }
 
+    @After
+    public void cleanup() {
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
+
     @Test
     public void shouldLeftJoinWithStream() {
         stream
@@ -100,16 +109,22 @@ public void shouldInnerJoinWithStream() {
     }
 
     private void verifyJoin(final Map expected) {
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "global-ktable-joins-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+
+        driver = new TopologyTestDriver(builder.build(), props);
+
         // write some data to the global table
-        driver.process(globalTopic, "a", "A");
-        driver.process(globalTopic, "b", "B");
+        driver.pipeInput(recordFactory.create(globalTopic, "a", "A"));
+        driver.pipeInput(recordFactory.create(globalTopic, "b", "B"));
         //write some data to the stream
-        driver.process(streamTopic, "1", "a");
-        driver.process(streamTopic, "2", "b");
-        driver.process(streamTopic, "3", "c");
-        driver.flushState();
+        driver.pipeInput(recordFactory.create(streamTopic, "1", "a"));
+        driver.pipeInput(recordFactory.create(streamTopic, "2", "b"));
+        driver.pipeInput(recordFactory.create(streamTopic, "3", "c"));
 
         assertEquals(expected, results);
     }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java
index b9ba60894973e..76ca495c2325b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java
@@ -30,11 +30,9 @@
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.test.KStreamTestDriver;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockTimestampExtractor;
 import org.apache.kafka.test.MockValueJoiner;
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -58,8 +56,6 @@ public class InternalStreamsBuilderTest {
     private static final String APP_ID = "app-id";
 
     private final InternalStreamsBuilder builder = new InternalStreamsBuilder(new InternalTopologyBuilder());
-
-    private KStreamTestDriver driver = null;
     private final ConsumedInternal consumed = new ConsumedInternal<>();
     private final String storePrefix = "prefix-";
     private MaterializedInternal> materialized
@@ -70,14 +66,6 @@ public void setUp() {
         builder.internalTopologyBuilder.setApplicationId(APP_ID);
     }
 
-    @After
-    public void cleanup() {
-        if (driver != null) {
-            driver.close();
-        }
-        driver = null;
-    }
-
     @Test
     public void testNewName() {
         assertEquals("X-0000000000", builder.newProcessorName("X-"));
@@ -105,7 +93,7 @@ public void testNewStoreName() {
     }
 
     @Test
-    public void shouldHaveCorrectSourceTopicsForTableFromMergedStream() throws Exception {
+    public void shouldHaveCorrectSourceTopicsForTableFromMergedStream() {
         final String topic1 = "topic-1";
         final String topic2 = "topic-2";
         final String topic3 = "topic-3";
@@ -138,7 +126,7 @@ public boolean test(final String key, final String value) {
     }
 
     @Test
-    public void shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable() throws Exception {
+    public void shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable() {
         KTable table1 = builder.table("topic2",
                                       consumed,
                                       new MaterializedInternal<>(
@@ -158,7 +146,7 @@ public void shouldStillMaterializeSourceKTableIfMaterializedIsntQueryable() thro
     }
     
     @Test
-    public void shouldBuildGlobalTableWithNonQueryableStoreName() throws Exception {
+    public void shouldBuildGlobalTableWithNonQueryableStoreName() {
         final GlobalKTable table1 = builder.globalTable(
             "topic2",
             consumed,
@@ -171,7 +159,7 @@ public void shouldBuildGlobalTableWithNonQueryableStoreName() throws Exception {
     }
 
     @Test
-    public void shouldBuildGlobalTableWithQueryaIbleStoreName() throws Exception {
+    public void shouldBuildGlobalTableWithQueryaIbleStoreName() {
         final GlobalKTable table1 = builder.globalTable(
             "topic2",
             consumed,
@@ -184,7 +172,7 @@ public void shouldBuildGlobalTableWithQueryaIbleStoreName() throws Exception {
     }
 
     @Test
-    public void shouldBuildSimpleGlobalTableTopology() throws Exception {
+    public void shouldBuildSimpleGlobalTableTopology() {
         builder.globalTable("table",
                             consumed,
                             new MaterializedInternal<>(
@@ -199,7 +187,7 @@ public void shouldBuildSimpleGlobalTableTopology() throws Exception {
         assertEquals("globalTable", stateStores.get(0).name());
     }
 
-    private void doBuildGlobalTopologyWithAllGlobalTables() throws Exception {
+    private void doBuildGlobalTopologyWithAllGlobalTables() {
         final ProcessorTopology topology = builder.internalTopologyBuilder.buildGlobalStateTopology();
 
         final List stateStores = topology.globalStateStores();
@@ -210,7 +198,7 @@ private void doBuildGlobalTopologyWithAllGlobalTables() throws Exception {
     }
 
     @Test
-    public void shouldBuildGlobalTopologyWithAllGlobalTables() throws Exception {
+    public void shouldBuildGlobalTopologyWithAllGlobalTables() {
         builder.globalTable("table",
                             consumed,
                             new MaterializedInternal<>(
@@ -224,7 +212,7 @@ public void shouldBuildGlobalTopologyWithAllGlobalTables() throws Exception {
     }
 
     @Test
-    public void shouldAddGlobalTablesToEachGroup() throws Exception {
+    public void shouldAddGlobalTablesToEachGroup() {
         final String one = "globalTable";
         final String two = "globalTable2";
 
@@ -269,7 +257,7 @@ public String apply(final String key, final String value) {
     }
 
     @Test
-    public void shouldMapStateStoresToCorrectSourceTopics() throws Exception {
+    public void shouldMapStateStoresToCorrectSourceTopics() {
         final KStream playEvents = builder.stream(Collections.singleton("events"), consumed);
 
         final MaterializedInternal> materialized
@@ -367,14 +355,14 @@ public void shouldAddRegexTopicToLatestAutoOffsetResetList() {
     }
 
     @Test
-    public void shouldHaveNullTimestampExtractorWhenNoneSupplied() throws Exception {
+    public void shouldHaveNullTimestampExtractorWhenNoneSupplied() {
         builder.stream(Collections.singleton("topic"), consumed);
         final ProcessorTopology processorTopology = builder.internalTopologyBuilder.build(null);
         assertNull(processorTopology.source("topic").getTimestampExtractor());
     }
 
     @Test
-    public void shouldUseProvidedTimestampExtractor() throws Exception {
+    public void shouldUseProvidedTimestampExtractor() {
         final ConsumedInternal consumed = new ConsumedInternal<>(Consumed.with(new MockTimestampExtractor()));
         builder.stream(Collections.singleton("topic"), consumed);
         final ProcessorTopology processorTopology = builder.internalTopologyBuilder.build(null);
@@ -382,14 +370,14 @@ public void shouldUseProvidedTimestampExtractor() throws Exception {
     }
 
     @Test
-    public void ktableShouldHaveNullTimestampExtractorWhenNoneSupplied() throws Exception {
+    public void ktableShouldHaveNullTimestampExtractorWhenNoneSupplied() {
         builder.table("topic", consumed, materialized);
         final ProcessorTopology processorTopology = builder.internalTopologyBuilder.build(null);
         assertNull(processorTopology.source("topic").getTimestampExtractor());
     }
 
     @Test
-    public void ktableShouldUseProvidedTimestampExtractor() throws Exception {
+    public void ktableShouldUseProvidedTimestampExtractor() {
         final ConsumedInternal consumed = new ConsumedInternal<>(Consumed.with(new MockTimestampExtractor()));
         builder.table("topic", consumed, materialized);
         final ProcessorTopology processorTopology = builder.internalTopologyBuilder.build(null);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
index c74d0dce44955..b9ca30f48cc39 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
@@ -20,10 +20,13 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.errors.InvalidTopicException;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.Aggregator;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.Initializer;
@@ -43,13 +46,13 @@
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.SessionStore;
 import org.apache.kafka.streams.state.WindowStore;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockReducer;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -57,6 +60,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -72,13 +76,30 @@ public class KGroupedStreamImplTest {
     private static final String INVALID_STORE_NAME = "~foo bar~";
     private final StreamsBuilder builder = new StreamsBuilder();
     private KGroupedStream groupedStream;
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
 
     @Before
     public void before() {
         final KStream stream = builder.stream(TOPIC, Consumed.with(Serdes.String(), Serdes.String()));
         groupedStream = stream.groupByKey(Serialized.with(Serdes.String(), Serdes.String()));
+
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kgrouped-stream-impl-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @SuppressWarnings("deprecation")
@@ -203,20 +224,13 @@ public void shouldNotHaveNullStoreSupplierOnWindowedAggregate() {
     }
 
     private void doAggregateSessionWindows(final Map, Integer> results) {
-        driver.setUp(builder, TestUtils.tempDirectory());
-        driver.setTime(10);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(15);
-        driver.process(TOPIC, "2", "2");
-        driver.setTime(30);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(70);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(90);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(100);
-        driver.process(TOPIC, "1", "1");
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 10));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "2", 15));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 30));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 70));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 90));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 100));
         assertEquals(Integer.valueOf(2), results.get(new Windowed<>("1", new SessionWindow(10, 30))));
         assertEquals(Integer.valueOf(1), results.get(new Windowed<>("2", new SessionWindow(15, 15))));
         assertEquals(Integer.valueOf(3), results.get(new Windowed<>("1", new SessionWindow(70, 100))));
@@ -284,20 +298,13 @@ public void apply(final Windowed key, final Integer value) {
     }
 
     private void doCountSessionWindows(final Map, Long> results) {
-        driver.setUp(builder, TestUtils.tempDirectory());
-        driver.setTime(10);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(15);
-        driver.process(TOPIC, "2", "2");
-        driver.setTime(30);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(70);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(90);
-        driver.process(TOPIC, "1", "1");
-        driver.setTime(100);
-        driver.process(TOPIC, "1", "1");
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 10));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "2", 15));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 30));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 70));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 90));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "1", 100));
         assertEquals(Long.valueOf(2), results.get(new Windowed<>("1", new SessionWindow(10, 30))));
         assertEquals(Long.valueOf(1), results.get(new Windowed<>("2", new SessionWindow(15, 15))));
         assertEquals(Long.valueOf(3), results.get(new Windowed<>("1", new SessionWindow(70, 100))));
@@ -334,20 +341,13 @@ public void apply(final Windowed key, final Long value) {
     }
 
     private void doReduceSessionWindows(final Map, String> results) {
-        driver.setUp(builder, TestUtils.tempDirectory());
-        driver.setTime(10);
-        driver.process(TOPIC, "1", "A");
-        driver.setTime(15);
-        driver.process(TOPIC, "2", "Z");
-        driver.setTime(30);
-        driver.process(TOPIC, "1", "B");
-        driver.setTime(70);
-        driver.process(TOPIC, "1", "A");
-        driver.setTime(90);
-        driver.process(TOPIC, "1", "B");
-        driver.setTime(100);
-        driver.process(TOPIC, "1", "C");
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 10));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "Z", 15));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "B", 30));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 70));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "B", 90));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "C", 100));
         assertEquals("A:B", results.get(new Windowed<>("1", new SessionWindow(10, 30))));
         assertEquals("Z", results.get(new Windowed<>("2", new SessionWindow(15, 15))));
         assertEquals("A:B:C", results.get(new Windowed<>("1", new SessionWindow(70, 100))));
@@ -556,8 +556,7 @@ public void shouldCountAndMaterializeResults() {
 
         processData();
 
-        @SuppressWarnings("unchecked") final KeyValueStore count =
-            (KeyValueStore) driver.allStateStores().get("count");
+        final KeyValueStore count = driver.getKeyValueStore("count");
 
         assertThat(count.get("1"), equalTo(3L));
         assertThat(count.get("2"), equalTo(1L));
@@ -571,10 +570,10 @@ public void shouldLogAndMeasureSkipsInAggregate() {
         processData();
         LogCaptureAppender.unregister(appender);
 
-        final Map metrics = driver.context().metrics().metrics();
+        final Map metrics = driver.metrics();
         assertEquals(1.0, getMetricByName(metrics, "skipped-records-total", "stream-metrics").metricValue());
         assertNotEquals(0.0, getMetricByName(metrics, "skipped-records-rate", "stream-metrics").metricValue());
-        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[3] value=[null] topic=[topic] partition=[-1] offset=[-1]"));
+        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[3] value=[null] topic=[topic] partition=[0] offset=[6]"));
     }
 
 
@@ -589,7 +588,7 @@ public void shouldReduceAndMaterializeResults() {
 
         processData();
 
-        final KeyValueStore reduced = (KeyValueStore) driver.allStateStores().get("reduce");
+        final KeyValueStore reduced = driver.getKeyValueStore("reduce");
 
         assertThat(reduced.get("1"), equalTo("A+C+D"));
         assertThat(reduced.get("2"), equalTo("B"));
@@ -609,10 +608,10 @@ public void shouldLogAndMeasureSkipsInReduce() {
         processData();
         LogCaptureAppender.unregister(appender);
 
-        final Map metrics = driver.context().metrics().metrics();
+        final Map metrics = driver.metrics();
         assertEquals(1.0, getMetricByName(metrics, "skipped-records-total", "stream-metrics").metricValue());
         assertNotEquals(0.0, getMetricByName(metrics, "skipped-records-rate", "stream-metrics").metricValue());
-        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[3] value=[null] topic=[topic] partition=[-1] offset=[-1]"));
+        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[3] value=[null] topic=[topic] partition=[0] offset=[6]"));
     }
 
 
@@ -628,7 +627,7 @@ public void shouldAggregateAndMaterializeResults() {
 
         processData();
 
-        final KeyValueStore aggregate = (KeyValueStore) driver.allStateStores().get("aggregate");
+        final KeyValueStore aggregate = driver.getKeyValueStore("aggregate");
 
         assertThat(aggregate.get("1"), equalTo("0+A+C+D"));
         assertThat(aggregate.get("2"), equalTo("0+B"));
@@ -658,29 +657,25 @@ public void apply(final String key, final String value) {
     }
 
     private void processData() {
-        driver.setUp(builder, TestUtils.tempDirectory(), Serdes.String(), Serdes.String(), 0);
-        driver.setTime(0);
-        driver.process(TOPIC, "1", "A");
-        driver.process(TOPIC, "2", "B");
-        driver.process(TOPIC, "1", "C");
-        driver.process(TOPIC, "1", "D");
-        driver.process(TOPIC, "3", "E");
-        driver.process(TOPIC, "3", "F");
-        driver.process(TOPIC, "3", null);
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "A"));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "B"));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "C"));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "D"));
+        driver.pipeInput(recordFactory.create(TOPIC, "3", "E"));
+        driver.pipeInput(recordFactory.create(TOPIC, "3", "F"));
+        driver.pipeInput(recordFactory.create(TOPIC, "3", null));
     }
 
     private void doCountWindowed(final List, Long>> results) {
-        driver.setUp(builder, TestUtils.tempDirectory(), 0);
-        driver.setTime(0);
-        driver.process(TOPIC, "1", "A");
-        driver.process(TOPIC, "2", "B");
-        driver.process(TOPIC, "3", "C");
-        driver.setTime(500);
-        driver.process(TOPIC, "1", "A");
-        driver.process(TOPIC, "1", "A");
-        driver.process(TOPIC, "2", "B");
-        driver.process(TOPIC, "2", "B");
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 0));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 0));
+        driver.pipeInput(recordFactory.create(TOPIC, "3", "C", 0));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 500));
+        driver.pipeInput(recordFactory.create(TOPIC, "1", "A", 500));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 500));
+        driver.pipeInput(recordFactory.create(TOPIC, "2", "B", 500));
         assertThat(results, equalTo(Arrays.asList(
             KeyValue.pair(new Windowed<>("1", new TimeWindow(0, 500)), 1L),
             KeyValue.pair(new Windowed<>("2", new TimeWindow(0, 500)), 1L),
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java
index 3bbd7e2fa5009..742f349657980 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java
@@ -17,11 +17,15 @@
 package org.apache.kafka.streams.kstream.internals;
 
 import org.apache.kafka.common.errors.InvalidTopicException;
+import org.apache.kafka.common.serialization.DoubleSerializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KGroupedTable;
 import org.apache.kafka.streams.kstream.KTable;
@@ -30,18 +34,19 @@
 import org.apache.kafka.streams.kstream.Serialized;
 import org.apache.kafka.streams.processor.StateStoreSupplier;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockReducer;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -55,14 +60,29 @@ public class KGroupedTableImplTest {
     private final StreamsBuilder builder = new StreamsBuilder();
     private static final String INVALID_STORE_NAME = "~foo bar~";
     private KGroupedTable groupedTable;
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
     private final String topic = "input";
 
     @Before
     public void before() {
         groupedTable = builder.table("blah", Consumed.with(Serdes.String(), Serdes.String()))
                 .groupBy(MockMapper.selectValueKeyValueMapper());
+
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kgrouped-table-impl-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test
@@ -122,6 +142,7 @@ public void shouldNotAllowNullStoreSupplierOnReduce() {
 
     private void doShouldReduce(final KTable reduced, final String topic) {
         final Map results = new HashMap<>();
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new DoubleSerializer());
         reduced.foreach(new ForeachAction() {
             @Override
             public void apply(final String key, final Integer value) {
@@ -129,20 +150,17 @@ public void apply(final String key, final Integer value) {
             }
         });
 
-        driver.setUp(builder, TestUtils.tempDirectory(), Serdes.String(), Serdes.Integer());
-        driver.setTime(10L);
-        driver.process(topic, "A", 1.1);
-        driver.process(topic, "B", 2.2);
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(topic, "A", 1.1, 10));
+        driver.pipeInput(recordFactory.create(topic, "B", 2.2, 10));
 
         assertEquals(Integer.valueOf(1), results.get("A"));
         assertEquals(Integer.valueOf(2), results.get("B"));
 
-        driver.process(topic, "A", 2.6);
-        driver.process(topic, "B", 1.3);
-        driver.process(topic, "A", 5.7);
-        driver.process(topic, "B", 6.2);
-        driver.flushState();
+        driver.pipeInput(recordFactory.create(topic, "A", 2.6, 10));
+        driver.pipeInput(recordFactory.create(topic, "B", 1.3, 10));
+        driver.pipeInput(recordFactory.create(topic, "A", 5.7, 10));
+        driver.pipeInput(recordFactory.create(topic, "B", 6.2, 10));
 
         assertEquals(Integer.valueOf(5), results.get("A"));
         assertEquals(Integer.valueOf(6), results.get("B"));
@@ -212,7 +230,7 @@ public KeyValue apply(String key, Number value) {
                                 .withValueSerde(Serdes.Integer()));
 
         doShouldReduce(reduced, topic);
-        final KeyValueStore reduce = (KeyValueStore) driver.allStateStores().get("reduce");
+        final KeyValueStore reduce = (KeyValueStore) driver.getStateStore("reduce");
         assertThat(reduce.get("A"), equalTo(5));
         assertThat(reduce.get("B"), equalTo(6));
     }
@@ -229,7 +247,7 @@ public void shouldCountAndMaterializeResults() {
                                .withValueSerde(Serdes.Long()));
 
         processData(topic);
-        final KeyValueStore counts = (KeyValueStore) driver.allStateStores().get("count");
+        final KeyValueStore counts = driver.getKeyValueStore("count");
         assertThat(counts.get("1"), equalTo(3L));
         assertThat(counts.get("2"), equalTo(2L));
     }
@@ -249,7 +267,7 @@ public void shouldAggregateAndMaterializeResults() {
                                    .withKeySerde(Serdes.String()));
 
         processData(topic);
-        final KeyValueStore aggregate = (KeyValueStore) driver.allStateStores().get("aggregate");
+        final KeyValueStore aggregate = (KeyValueStore) driver.getStateStore("aggregate");
         assertThat(aggregate.get("1"), equalTo("0+1+1+1"));
         assertThat(aggregate.get("2"), equalTo("0+2+2"));
     }
@@ -310,13 +328,12 @@ public void shouldThrowNullPointerOnAggregateWhenMaterializedIsNull() {
     }
 
     private void processData(final String topic) {
-        driver.setUp(builder, TestUtils.tempDirectory(), Serdes.String(), Serdes.Integer());
-        driver.setTime(0L);
-        driver.process(topic, "A", "1");
-        driver.process(topic, "B", "1");
-        driver.process(topic, "C", "1");
-        driver.process(topic, "D", "2");
-        driver.process(topic, "E", "2");
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props);
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+        driver.pipeInput(recordFactory.create(topic, "A", "1"));
+        driver.pipeInput(recordFactory.create(topic, "B", "1"));
+        driver.pipeInput(recordFactory.create(topic, "C", "1"));
+        driver.pipeInput(recordFactory.create(topic, "D", "2"));
+        driver.pipeInput(recordFactory.create(topic, "E", "2"));
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java
index 702631a93e7d4..a70bc37942ea0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamBranchTest.java
@@ -16,26 +16,51 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Predicate;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.lang.reflect.Array;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
 public class KStreamBranchTest {
 
-    private String topicName = "topic";
+    private final String topicName = "topic";
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void before() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-branch-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
 
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @SuppressWarnings("unchecked")
     @Test
@@ -78,9 +103,9 @@ public boolean test(Integer key, String value) {
             branches[i].process(processors[i]);
         }
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, "V" + expectedKey);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
         }
 
         assertEquals(3, processors[0].processed.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java
index f1a6152852fd9..a67d688e27821 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFilterTest.java
@@ -16,25 +16,51 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Predicate;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Properties;
+
 import static org.junit.Assert.assertEquals;
 
 public class KStreamFilterTest {
 
-    private String topicName = "topic";
-  
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final String topicName = "topic";
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void before() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-filter-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
 
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
+  
     private Predicate isMultipleOfThree = new Predicate() {
         @Override
         public boolean test(Integer key, String value) {
@@ -54,9 +80,9 @@ public void testFilter() {
         stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
         stream.filter(isMultipleOfThree).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, "V" + expectedKey);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
         }
 
         assertEquals(2, processor.processed.size());
@@ -74,9 +100,9 @@ public void testFilterNot() {
         stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
         stream.filterNot(isMultipleOfThree).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, "V" + expectedKey);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
         }
 
         assertEquals(5, processor.processed.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java
index 59dad4706d4e5..e414218898d17 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapTest.java
@@ -16,27 +16,52 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
 public class KStreamFlatMapTest {
 
     private String topicName = "topic";
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
 
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    @Before
+    public void before() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-flat-map-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testFlatMap() {
@@ -63,9 +88,9 @@ public Iterable> apply(Number key, Object value) {
         stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
         stream.flatMap(mapper).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, "V" + expectedKey);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
         }
 
         assertEquals(6, processor.processed.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
index ecfa3aac13fc3..14213c9966866 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamFlatMapValuesTest.java
@@ -16,27 +16,51 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueMapper;
 import org.apache.kafka.streams.kstream.ValueMapperWithKey;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
+import java.util.Properties;
 
 import static org.junit.Assert.assertArrayEquals;
 
 public class KStreamFlatMapValuesTest {
 
     private String topicName = "topic";
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void before() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-flat-map-values-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
 
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testFlatMapValues() {
@@ -59,9 +83,10 @@ public Iterable apply(Number value) {
         final MockProcessorSupplier processor = new MockProcessorSupplier<>();
         stream.flatMapValues(mapper).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (final int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, expectedKey);
+            // passing the timestamp to recordFactory.create to disambiguate the call
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey, 0L));
         }
 
         String[] expected = {"0:v0", "0:V0", "1:v1", "1:V1", "2:v2", "2:V2", "3:v3", "3:V3"};
@@ -92,9 +117,10 @@ public Iterable apply(final Integer readOnlyKey, final Number value) {
 
         stream.flatMapValues(mapper).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (final int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, expectedKey);
+            // passing the timestamp to recordFactory.create to disambiguate the call
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey, 0L));
         }
 
         String[] expected = {"0:v0", "0:k0", "1:v1", "1:k1", "2:v2", "2:k2", "3:v3", "3:k3"};
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
index e8854fc4e41f6..b975c96e7d1e0 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamForeachTest.java
@@ -16,32 +16,58 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.test.KStreamTestDriver;
-import org.junit.Rule;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
 public class KStreamForeachTest {
 
-    final private String topicName = "topic";
+    private final String topicName = "topic";
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private Properties props = new Properties();
 
-    final private Serde intSerde = Serdes.Integer();
-    final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    @Before
+    public void before() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-foreach-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
+
+    private final Serde intSerde = Serdes.Integer();
+    private final Serde stringSerde = Serdes.String();
 
     @Test
     public void testForeach() {
@@ -75,9 +101,9 @@ public void apply(Integer key, String value) {
         stream.foreach(action);
 
         // Then
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (KeyValue record: inputRecords) {
-            driver.process(topicName, record.key, record.value);
+            driver.pipeInput(recordFactory.create(topicName, record.key, record.value));
         }
 
         assertEquals(expectedRecords.size(), actualRecords.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java
index d1d048bc057eb..2936f5fb3c420 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableJoinTest.java
@@ -16,46 +16,46 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.GlobalKTable;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 
 public class KStreamGlobalKTableJoinTest {
 
-    final private String streamTopic = "streamTopic";
-    final private String globalTableTopic = "globalTableTopic";
-
-    final private Serde intSerde = Serdes.Integer();
-    final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
-    private File stateDir = null;
+    private final String streamTopic = "streamTopic";
+    private final String globalTableTopic = "globalTableTopic";
+    private final Serde intSerde = Serdes.Integer();
+    private final Serde stringSerde = Serdes.String();
+    private TopologyTestDriver driver;
     private MockProcessorSupplier processor;
     private final int[] expectedKeys = {0, 1, 2, 3};
     private StreamsBuilder builder;
 
     @Before
     public void setUp() throws IOException {
-        stateDir = TestUtils.tempDirectory("kafka-test");
 
         builder = new StreamsBuilder();
         final KStream stream;
@@ -78,29 +78,46 @@ public String apply(final Integer key, final String value) {
         };
         stream.join(table, keyMapper, MockValueJoiner.TOSTRING_JOINER).process(processor);
 
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-global-ktable-join-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+
+        driver = new TopologyTestDriver(builder.build(), props);
+    }
+
+    @After
+    public void cleanup() {
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     private void pushToStream(final int messageCount, final String valuePrefix, final boolean includeForeignKey) {
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
             String value = valuePrefix + expectedKeys[i];
             if (includeForeignKey) {
                 value = value + ",FKey" + expectedKeys[i];
             }
-            driver.process(streamTopic, expectedKeys[i], value);
+            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], value));
         }
     }
 
     private void pushToGlobalTable(final int messageCount, final String valuePrefix) {
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
-            driver.process(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]));
         }
     }
 
     private void pushNullValueToGlobalTable(final int messageCount) {
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
-            driver.process(globalTableTopic, "FKey" + expectedKeys[i], null);
+            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], null));
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java
index 8b7dd4273871f..88821130d3657 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamGlobalKTableLeftJoinTest.java
@@ -16,25 +16,28 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.GlobalKTable;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -43,19 +46,15 @@ public class KStreamGlobalKTableLeftJoinTest {
 
     final private String streamTopic = "streamTopic";
     final private String globalTableTopic = "globalTableTopic";
-
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
-    private File stateDir = null;
+    private TopologyTestDriver driver;
     private MockProcessorSupplier processor;
     private final int[] expectedKeys = {0, 1, 2, 3};
     private StreamsBuilder builder;
 
     @Before
     public void setUp() throws IOException {
-        stateDir = TestUtils.tempDirectory("kafka-test");
 
         builder = new StreamsBuilder();
         final KStream stream;
@@ -78,29 +77,38 @@ public String apply(final Integer key, final String value) {
         };
         stream.leftJoin(table, keyMapper, MockValueJoiner.TOSTRING_JOINER).process(processor);
 
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-global-ktable-left-join-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+
+        driver = new TopologyTestDriver(builder.build(), props);
     }
 
     private void pushToStream(final int messageCount, final String valuePrefix, final boolean includeForeignKey) {
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
             String value = valuePrefix + expectedKeys[i];
             if (includeForeignKey) {
                 value = value + ",FKey" + expectedKeys[i];
             }
-            driver.process(streamTopic, expectedKeys[i], value);
+            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], value));
         }
     }
 
     private void pushToGlobalTable(final int messageCount, final String valuePrefix) {
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
-            driver.process(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], valuePrefix + expectedKeys[i]));
         }
     }
 
     private void pushNullValueToGlobalTable(final int messageCount) {
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
         for (int i = 0; i < messageCount; i++) {
-            driver.process(globalTableTopic, "FKey" + expectedKeys[i], null);
+            driver.pipeInput(recordFactory.create(globalTableTopic, "FKey" + expectedKeys[i], null));
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
index ef65bb32b36d6..f397246c7b68d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java
@@ -18,11 +18,14 @@
 
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.errors.TopologyException;
 import org.apache.kafka.streams.kstream.GlobalKTable;
 import org.apache.kafka.streams.kstream.JoinWindows;
@@ -42,16 +45,18 @@
 import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
 import org.apache.kafka.streams.processor.internals.ProcessorTopology;
 import org.apache.kafka.streams.processor.internals.SourceNode;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockMapper;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
@@ -71,13 +76,29 @@ public class KStreamImplTest {
     private StreamsBuilder builder;
     private final Consumed consumed = Consumed.with(stringSerde, stringSerde);
 
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
 
     @Before
     public void before() {
         builder = new StreamsBuilder();
         testStream = builder.stream("source");
+
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-impl-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test
@@ -204,8 +225,8 @@ public void shouldSendDataThroughTopicUsingProduced() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         stream.through("through-topic", Produced.with(stringSerde, stringSerde)).process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.process(input, "a", "b");
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(input, "a", "b"));
         assertThat(processorSupplier.processed, equalTo(Collections.singletonList("a:b")));
     }
 
@@ -218,8 +239,8 @@ public void shouldSendDataToTopicUsingProduced() {
         stream.to("to-topic", Produced.with(stringSerde, stringSerde));
         builder.stream("to-topic", consumed).process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.process(input, "e", "f");
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create(input, "e", "f"));
         assertThat(processorSupplier.processed, equalTo(Collections.singletonList("e:f")));
     }
 
@@ -501,13 +522,12 @@ public void shouldMergeTwoStreams() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         merged.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process(topic1, "A", "aa");
-        driver.process(topic2, "B", "bb");
-        driver.process(topic2, "C", "cc");
-        driver.process(topic1, "D", "dd");
+        driver.pipeInput(recordFactory.create(topic1, "A", "aa"));
+        driver.pipeInput(recordFactory.create(topic2, "B", "bb"));
+        driver.pipeInput(recordFactory.create(topic2, "C", "cc"));
+        driver.pipeInput(recordFactory.create(topic1, "D", "dd"));
 
         assertEquals(Utils.mkList("A:aa", "B:bb", "C:cc", "D:dd"), processorSupplier.processed);
     }
@@ -528,17 +548,16 @@ public void shouldMergeMultipleStreams() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         merged.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process(topic1, "A", "aa");
-        driver.process(topic2, "B", "bb");
-        driver.process(topic3, "C", "cc");
-        driver.process(topic4, "D", "dd");
-        driver.process(topic4, "E", "ee");
-        driver.process(topic3, "F", "ff");
-        driver.process(topic2, "G", "gg");
-        driver.process(topic1, "H", "hh");
+        driver.pipeInput(recordFactory.create(topic1, "A", "aa"));
+        driver.pipeInput(recordFactory.create(topic2, "B", "bb"));
+        driver.pipeInput(recordFactory.create(topic3, "C", "cc"));
+        driver.pipeInput(recordFactory.create(topic4, "D", "dd"));
+        driver.pipeInput(recordFactory.create(topic4, "E", "ee"));
+        driver.pipeInput(recordFactory.create(topic3, "F", "ff"));
+        driver.pipeInput(recordFactory.create(topic2, "G", "gg"));
+        driver.pipeInput(recordFactory.create(topic1, "H", "hh"));
 
         assertEquals(Utils.mkList("A:aa", "B:bb", "C:cc", "D:dd", "E:ee", "F:ff", "G:gg", "H:hh"),
                      processorSupplier.processed);
@@ -551,14 +570,13 @@ public void shouldProcessFromSourceThatMatchPattern() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         pattern2Source.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process("topic-3", "A", "aa");
-        driver.process("topic-4", "B", "bb");
-        driver.process("topic-5", "C", "cc");
-        driver.process("topic-6", "D", "dd");
-        driver.process("topic-7", "E", "ee");
+        driver.pipeInput(recordFactory.create("topic-3", "A", "aa"));
+        driver.pipeInput(recordFactory.create("topic-4", "B", "bb"));
+        driver.pipeInput(recordFactory.create("topic-5", "C", "cc"));
+        driver.pipeInput(recordFactory.create("topic-6", "D", "dd"));
+        driver.pipeInput(recordFactory.create("topic-7", "E", "ee"));
 
         assertEquals(Utils.mkList("A:aa", "B:bb", "C:cc", "D:dd", "E:ee"),
                 processorSupplier.processed);
@@ -576,14 +594,13 @@ public void shouldProcessFromSourcesThatMatchMultiplePattern() {
         final MockProcessorSupplier processorSupplier = new MockProcessorSupplier<>();
         merged.process(processorSupplier);
 
-        driver.setUp(builder);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
-        driver.process("topic-3", "A", "aa");
-        driver.process("topic-4", "B", "bb");
-        driver.process("topic-A", "C", "cc");
-        driver.process("topic-Z", "D", "dd");
-        driver.process(topic3, "E", "ee");
+        driver.pipeInput(recordFactory.create("topic-3", "A", "aa"));
+        driver.pipeInput(recordFactory.create("topic-4", "B", "bb"));
+        driver.pipeInput(recordFactory.create("topic-A", "C", "cc"));
+        driver.pipeInput(recordFactory.create("topic-Z", "D", "dd"));
+        driver.pipeInput(recordFactory.create(topic3, "E", "ee"));
 
         assertEquals(Utils.mkList("A:aa", "B:bb", "C:cc", "D:dd", "E:ee"),
                 processorSupplier.processed);
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
index 4d2bce5bea01c..63a040acd5e1b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
@@ -16,30 +16,32 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.JoinWindows;
 import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueJoiner;
-import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
-import org.apache.kafka.test.InternalMockProcessorContext;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
@@ -55,14 +57,27 @@ public class KStreamKStreamJoinTest {
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
 
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
-    private File stateDir = null;
     private final Consumed consumed = Consumed.with(intSerde, stringSerde);
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
 
     @Before
     public void setUp() {
-        stateDir = TestUtils.tempDirectory("kafka-test");
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-kstream-join-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test
@@ -71,6 +86,7 @@ public void shouldLogAndMeterOnSkippedRecordsWithNullValue() {
 
         final KStream left = builder.stream("left", Consumed.with(stringSerde, intSerde));
         final KStream right = builder.stream("right", Consumed.with(stringSerde, intSerde));
+        final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new IntegerSerializer());
 
         left.join(
             right,
@@ -85,13 +101,13 @@ public Integer apply(final Integer value1, final Integer value2) {
         );
 
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        driver.setUp(builder, stateDir);
-        driver.process("left", "A", null);
+        driver = new TopologyTestDriver(builder.build(), props);
+        driver.pipeInput(recordFactory.create("left", "A", null));
         LogCaptureAppender.unregister(appender);
 
-        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[A] value=[null] topic=[left] partition=[-1] offset=[-1]"));
+        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[A] value=[null] topic=[left] partition=[0] offset=[0]"));
 
-        assertEquals(1.0, getMetricByName(driver.context().metrics().metrics(), "skipped-records-total", "stream-metrics").metricValue());
+        assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
     }
 
     @Test
@@ -120,8 +136,7 @@ public void testJoin() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props);
 
         // push two items to the primary stream. the other window is empty
         // w1 = {}
@@ -130,7 +145,7 @@ public void testJoin() {
         //     w2 = {}
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i]));
         }
 
         processor.checkAndClearProcessResult();
@@ -142,7 +157,7 @@ public void testJoin() {
         //     w2 = { 0:Y0, 1:Y1 }
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i]));
         }
 
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
@@ -153,8 +168,8 @@ public void testJoin() {
         // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 }
         //     w2 = { 0:Y0, 1:Y1 }
 
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "X" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey));
         }
 
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
@@ -165,8 +180,8 @@ public void testJoin() {
         // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 }
         //     w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3 }
 
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
@@ -177,8 +192,8 @@ public void testJoin() {
         // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3,  0:XX0, 1:XX1, 2:XX2, 3:XX3 }
         //     w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3 }
 
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3");
@@ -190,7 +205,7 @@ public void testJoin() {
         //     w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3, 0:YYY0, 1:YYY1 }
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "YYY" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "YYY" + expectedKeys[i]));
         }
 
         processor.checkAndClearProcessResult("0:X0+YYY0", "0:X0+YYY0", "0:XX0+YYY0", "1:X1+YYY1", "1:X1+YYY1", "1:XX1+YYY1");
@@ -222,8 +237,7 @@ public void testOuterJoin() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
 
         // push two items to the primary stream. the other window is empty.this should produce two items
         // w1 = {}
@@ -232,7 +246,7 @@ public void testOuterJoin() {
         //     w2 = {}
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i]));
         }
 
         processor.checkAndClearProcessResult("0:X0+null", "1:X1+null");
@@ -244,7 +258,7 @@ public void testOuterJoin() {
         //     w2 = { 0:Y0, 1:Y1 }
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i]));
         }
 
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
@@ -255,8 +269,8 @@ public void testOuterJoin() {
         // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 }
         //     w2 = { 0:Y0, 1:Y1 }
 
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "X" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "X" + expectedKey));
         }
 
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null", "3:X3+null");
@@ -267,8 +281,8 @@ public void testOuterJoin() {
         // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3 }
         //     w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 }
 
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
@@ -279,8 +293,8 @@ public void testOuterJoin() {
         // --> w1 = { 0:X0, 1:X1, 0:X0, 1:X1, 2:X2, 3:X3,  0:XX0, 1:XX1, 2:XX2, 3:XX3 }
         //     w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3 }
 
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3");
@@ -292,7 +306,7 @@ public void testOuterJoin() {
         //     w2 = { 0:Y0, 1:Y1, 0:YY0, 0:YY0, 1:YY1, 2:YY2, 3:YY3, 0:YYY0, 1:YYY1 }
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "YYY" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "YYY" + expectedKeys[i]));
         }
 
         processor.checkAndClearProcessResult("0:X0+YYY0", "0:X0+YYY0", "0:XX0+YYY0", "1:X1+YYY1", "1:X1+YYY1", "1:XX1+YYY1");
@@ -327,17 +341,15 @@ public void testWindowing() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir);
-
+        driver = new TopologyTestDriver(builder.build(), props, time);
 
         // push two items to the primary stream. the other window is empty. this should produce no items.
         // w1 = {}
         // w2 = {}
         // --> w1 = { 0:X0, 1:X1 }
         //     w2 = {}
-        setRecordContext(time, topic1);
         for (int i = 0; i < 2; i++) {
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time));
         }
 
         processor.checkAndClearProcessResult();
@@ -348,58 +360,53 @@ public void testWindowing() {
         // --> w1 = { 0:X0, 1:X1 }
         //     w2 = { 0:Y0, 1:Y1 }
 
-        setRecordContext(time, topic2);
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time));
         }
 
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
 
         // clear logically
         time = 1000L;
-        setRecordContext(time, topic1);
         for (int i = 0; i < expectedKeys.length; i++) {
-            setRecordContext(time + i, topic1);
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time + i));
         }
         processor.checkAndClearProcessResult();
 
         // gradually expires items in w1
         // w1 = { 0:X0, 1:X1, 2:X2, 3:X3 }
 
-        time = 1000 + 100L;
-        setRecordContext(time, topic2);
-
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 100L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
@@ -407,37 +414,36 @@ public void testWindowing() {
         // go back to the time before expiration
 
         time = 1000L - 100L - 1L;
-        setRecordContext(time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
@@ -445,8 +451,7 @@ public void testWindowing() {
         // clear (logically)
         time = 2000L;
         for (int i = 0; i < expectedKeys.length; i++) {
-            setRecordContext(time + i, topic2);
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time + i));
         }
 
         processor.checkAndClearProcessResult();
@@ -455,37 +460,36 @@ public void testWindowing() {
         // w2 = { 0:Y0, 1:Y1, 2:Y2, 3:Y3 }
 
         time = 2000L + 100L;
-        setRecordContext(time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("2:XX2+Y2", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
@@ -493,37 +497,36 @@ public void testWindowing() {
         // go back to the time before expiration
 
         time = 2000L - 100L - 1L;
-        setRecordContext(time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2");
 
-        setRecordContext(++time, topic1);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
@@ -560,85 +563,80 @@ public void testAsymmetricWindowingAfter() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir);
+        driver = new TopologyTestDriver(builder.build(), props, time);
 
         for (int i = 0; i < expectedKeys.length; i++) {
-            setRecordContext(time + i, topic1);
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time + i));
         }
         processor.checkAndClearProcessResult();
 
 
         time = 1000L - 1L;
-        setRecordContext(time, topic2);
-
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
         time = 1000 + 100L;
-        setRecordContext(time, topic2);
-
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
@@ -673,90 +671,82 @@ public void testAsymmetricWindowingBefore() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir);
+        driver = new TopologyTestDriver(builder.build(), props, time);
 
         for (int i = 0; i < expectedKeys.length; i++) {
-            setRecordContext(time + i, topic1);
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time + i));
         }
         processor.checkAndClearProcessResult();
 
 
         time = 1000L - 100L - 1L;
-
-        setRecordContext(time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
-        time = 1000L;
 
-        setRecordContext(time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time = 1000L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult("3:X3+YY3");
 
-        setRecordContext(++time, topic2);
-        for (final int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+        time += 1L;
+        for (int expectedKey : expectedKeys) {
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey, time));
         }
 
         processor.checkAndClearProcessResult();
     }
-
-    private void setRecordContext(final long time, final String topic) {
-        ((InternalMockProcessorContext) driver.context()).setRecordContext(new ProcessorRecordContext(time, 0, 0, topic));
-    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
index 465082b7aa796..cb1aaf160126e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamLeftJoinTest.java
@@ -16,29 +16,30 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.JoinWindows;
 import org.apache.kafka.streams.kstream.Joined;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
-import org.apache.kafka.test.KStreamTestDriver;
-import org.apache.kafka.test.InternalMockProcessorContext;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -50,15 +51,27 @@ public class KStreamKStreamLeftJoinTest {
 
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
-    private File stateDir = null;
     private final Consumed consumed = Consumed.with(intSerde, stringSerde);
-
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private Properties props = new Properties();
 
     @Before
-    public void setUp() throws IOException {
-        stateDir = TestUtils.tempDirectory("kafka-test");
+    public void setUp() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-kstream-left-join-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test
@@ -87,8 +100,7 @@ public void testLeftJoin() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir, Serdes.Integer(), Serdes.String());
-        driver.setTime(0L);
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
 
         // push two items to the primary stream. the other window is empty
         // w1 {}
@@ -97,9 +109,8 @@ public void testLeftJoin() {
         // --> w2 = {}
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i]));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:X0+null", "1:X1+null");
 
         // push two items to the other stream. this should produce two items.
@@ -109,9 +120,8 @@ public void testLeftJoin() {
         // --> w2 = { 0:Y0, 1:Y1 }
 
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i]));
         }
-        driver.flushState();
 
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
 
@@ -122,9 +132,8 @@ public void testLeftJoin() {
         // --> w2 = { 0:Y0, 1:Y1 }
 
         for (int i = 0; i < 3; i++) {
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i]));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null");
 
         // push all items to the other stream. this should produce 5 items
@@ -134,9 +143,8 @@ public void testLeftJoin() {
         // --> w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3}
 
         for (int expectedKey : expectedKeys) {
-            driver.process(topic2, expectedKey, "YY" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic2, expectedKey, "YY" + expectedKey));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2");
 
         // push all four items to the primary stream. this should produce six items.
@@ -146,9 +154,8 @@ public void testLeftJoin() {
         // --> w2 = { 0:Y0, 1:Y1, 0:YY0, 1:YY1, 2:YY2, 3:YY3}
 
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3");
     }
 
@@ -178,7 +185,7 @@ public void testWindowing() {
         assertEquals(1, copartitionGroups.size());
         assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
 
-        driver.setUp(builder, stateDir);
+        driver = new TopologyTestDriver(builder.build(), props, time);
 
         // push two items to the primary stream. the other window is empty. this should produce two items
         // w1 = {}
@@ -186,11 +193,9 @@ public void testWindowing() {
         // --> w1 = { 0:X0, 1:X1 }
         // --> w2 = {}
 
-        setRecordContext(time, topic1);
         for (int i = 0; i < 2; i++) {
-            driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic1, expectedKeys[i], "X" + expectedKeys[i], time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:X0+null", "1:X1+null");
 
         // push two items to the other stream. this should produce no items.
@@ -199,16 +204,13 @@ public void testWindowing() {
         // --> w1 = { 0:X0, 1:X1 }
         // --> w2 = { 0:Y0, 1:Y1 }
 
-        setRecordContext(time, topic2);
         for (int i = 0; i < 2; i++) {
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
 
         // clear logically
         time = 1000L;
-        setRecordContext(time, topic2);
 
         // push all items to the other stream. this should produce no items.
         // w1 = {}
@@ -216,10 +218,8 @@ public void testWindowing() {
         // --> w1 = {}
         // --> w2 = { 0:Y0, 1:Y1, 2:Y2, 3:Y3 }
         for (int i = 0; i < expectedKeys.length; i++) {
-            setRecordContext(time + i, topic2);
-            driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(topic2, expectedKeys[i], "Y" + expectedKeys[i], time + i));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult();
 
         // gradually expire items in window 2.
@@ -229,81 +229,65 @@ public void testWindowing() {
         // --> w2 = { 0:Y0, 1:Y1, 2:Y2, 3:Y3 }
 
         time = 1000L + 100L;
-        setRecordContext(time, topic1);
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+Y2", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+null", "3:XX3+Y3");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+null", "3:XX3+null");
 
         // go back to the time before expiration
 
         time = 1000L - 100L - 1L;
-        setRecordContext(time, topic1);
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+null", "1:XX1+null", "2:XX2+null", "3:XX3+null");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+null", "2:XX2+null", "3:XX3+null");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+null", "3:XX3+null");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+null");
 
-        setRecordContext(++time, topic1);
+        time += 1L;
         for (int expectedKey : expectedKeys) {
-            driver.process(topic1, expectedKey, "XX" + expectedKey);
+            driver.pipeInput(recordFactory.create(topic1, expectedKey, "XX" + expectedKey, time));
         }
-        driver.flushState();
         processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
     }
-
-    private void setRecordContext(final long time, final String topic) {
-        ((InternalMockProcessorContext) driver.context()).setRecordContext(new ProcessorRecordContext(time, 0, 0, topic));
-    }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java
index 18003852176ae..5b2a797b1e2cf 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableJoinTest.java
@@ -16,26 +16,29 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
@@ -45,24 +48,21 @@
 
 public class KStreamKTableJoinTest {
 
-    final private String streamTopic = "streamTopic";
-    final private String tableTopic = "tableTopic";
+    private final String streamTopic = "streamTopic";
+    private final String tableTopic = "tableTopic";
 
-    final private Serde intSerde = Serdes.Integer();
-    final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final Serde intSerde = Serdes.Integer();
+    private final Serde stringSerde = Serdes.String();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
     private MockProcessorSupplier processor;
     private final int[] expectedKeys = {0, 1, 2, 3};
     private StreamsBuilder builder;
 
     @Before
     public void setUp() {
-        final File stateDir = TestUtils.tempDirectory("kafka-test");
-
         builder = new StreamsBuilder();
 
-
         final KStream stream;
         final KTable table;
 
@@ -72,25 +72,31 @@ public void setUp() {
         table = builder.table(tableTopic, consumed);
         stream.join(table, MockValueJoiner.TOSTRING_JOINER).process(processor);
 
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-ktable-join-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
     }
 
     private void pushToStream(final int messageCount, final String valuePrefix) {
         for (int i = 0; i < messageCount; i++) {
-            driver.process(streamTopic, expectedKeys[i], valuePrefix + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], valuePrefix + expectedKeys[i]));
         }
     }
 
     private void pushToTable(final int messageCount, final String valuePrefix) {
         for (int i = 0; i < messageCount; i++) {
-            driver.process(tableTopic, expectedKeys[i], valuePrefix + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], valuePrefix + expectedKeys[i]));
         }
     }
 
     private void pushNullValueToTable() {
         for (int i = 0; i < 2; i++) {
-            driver.process(tableTopic, expectedKeys[i], null);
+            driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], null));
         }
     }
 
@@ -188,20 +194,20 @@ public void shouldClearTableEntryOnNullValueUpdates() {
     @Test
     public void shouldLogAndMeterWhenSkippingNullLeftKey() {
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        driver.process(streamTopic, null, "A");
+        driver.pipeInput(recordFactory.create(streamTopic, null, "A"));
         LogCaptureAppender.unregister(appender);
 
-        assertEquals(1.0, getMetricByName(driver.context().metrics().metrics(), "skipped-records-total", "stream-metrics").metricValue());
-        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[null] value=[A] topic=[streamTopic] partition=[-1] offset=[-1]"));
+        assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
+        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[null] value=[A] topic=[streamTopic] partition=[0] offset=[0]"));
     }
 
     @Test
     public void shouldLogAndMeterWhenSkippingNullLeftValue() {
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        driver.process(streamTopic, 1, null);
+        driver.pipeInput(recordFactory.create(streamTopic, 1, null));
         LogCaptureAppender.unregister(appender);
 
-        assertEquals(1.0, getMetricByName(driver.context().metrics().metrics(), "skipped-records-total", "stream-metrics").metricValue());
-        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[1] value=[null] topic=[streamTopic] partition=[-1] offset=[-1]"));
+        assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
+        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. key=[1] value=[null] topic=[streamTopic] partition=[0] offset=[0]"));
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java
index 7507d7a4def24..669f4c7712f0d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKTableLeftJoinTest.java
@@ -16,26 +16,28 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsBuilderTest;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Properties;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -47,17 +49,14 @@ public class KStreamKTableLeftJoinTest {
 
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
-    private File stateDir = null;
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
     private MockProcessorSupplier processor;
     private final int[] expectedKeys = {0, 1, 2, 3};
     private StreamsBuilder builder;
 
     @Before
-    public void setUp() throws IOException {
-        stateDir = TestUtils.tempDirectory("kafka-test");
-
+    public void setUp() {
         builder = new StreamsBuilder();
 
 
@@ -70,25 +69,31 @@ public void setUp() throws IOException {
         table = builder.table(tableTopic, consumed);
         stream.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).process(processor);
 
-        driver.setUp(builder, stateDir);
-        driver.setTime(0L);
+        final Properties props = new Properties();
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-ktable-left-join-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
     }
 
     private void pushToStream(final int messageCount, final String valuePrefix) {
         for (int i = 0; i < messageCount; i++) {
-            driver.process(streamTopic, expectedKeys[i], valuePrefix + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(streamTopic, expectedKeys[i], valuePrefix + expectedKeys[i]));
         }
     }
 
     private void pushToTable(final int messageCount, final String valuePrefix) {
         for (int i = 0; i < messageCount; i++) {
-            driver.process(tableTopic, expectedKeys[i], valuePrefix + expectedKeys[i]);
+            driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], valuePrefix + expectedKeys[i]));
         }
     }
 
     private void pushNullValueToTable(final int messageCount) {
         for (int i = 0; i < messageCount; i++) {
-            driver.process(tableTopic, expectedKeys[i], null);
+            driver.pipeInput(recordFactory.create(tableTopic, expectedKeys[i], null));
         }
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java
index eed7d7dda4092..bb22204946415 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapTest.java
@@ -16,18 +16,26 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Properties;
+
 import static org.junit.Assert.assertEquals;
 
 public class KStreamMapTest {
@@ -36,8 +44,27 @@ public class KStreamMapTest {
 
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-map-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testMap() {
@@ -59,9 +86,9 @@ public KeyValue apply(Integer key, String value) {
         processor = new MockProcessorSupplier<>();
         stream.map(mapper).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, "V" + expectedKey);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, "V" + expectedKey));
         }
 
         assertEquals(4, processor.processed.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java
index 2cedfb425bb83..17a13e0b82ebd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamMapValuesTest.java
@@ -16,18 +16,26 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueMapper;
 import org.apache.kafka.streams.kstream.ValueMapperWithKey;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Properties;
+
 import static org.junit.Assert.assertArrayEquals;
 
 public class KStreamMapValuesTest {
@@ -36,8 +44,27 @@ public class KStreamMapValuesTest {
 
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-map-values-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testFlatMapValues() {
@@ -58,9 +85,9 @@ public Integer apply(CharSequence value) {
         stream = builder.stream(topicName, Consumed.with(intSerde, stringSerde));
         stream.mapValues(mapper).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, Integer.toString(expectedKey));
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, Integer.toString(expectedKey)));
         }
         String[] expected = {"1:1", "10:2", "100:3", "1000:4"};
 
@@ -86,9 +113,9 @@ public Integer apply(final Integer readOnlyKey, final CharSequence value) {
         stream = builder.stream(topicName, Consumed.with(intSerde, stringSerde));
         stream.mapValues(mapper).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, Integer.toString(expectedKey));
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, Integer.toString(expectedKey)));
         }
         String[] expected = {"1:2", "10:12", "100:103", "1000:1004"};
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java
index 215b0c36a3940..2c6ff81b8f686 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamPeekTest.java
@@ -16,20 +16,26 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.test.KStreamTestDriver;
-
-import org.junit.Rule;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -39,8 +45,27 @@ public class KStreamPeekTest {
     private final String topicName = "topic";
     private final Serde intSerd = Serdes.Integer();
     private final Serde stringSerd = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-peek-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void shouldObserveStreamElements() {
@@ -49,11 +74,11 @@ public void shouldObserveStreamElements() {
         final List> peekObserved = new ArrayList<>(), streamObserved = new ArrayList<>();
         stream.peek(collect(peekObserved)).foreach(collect(streamObserved));
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
         final List> expected = new ArrayList<>();
         for (int key = 0; key < 32; key++) {
             final String value = "V" + key;
-            driver.process(topicName, key, value);
+            driver.pipeInput(recordFactory.create(topicName, key, value));
             expected.add(new KeyValue<>(key, value));
         }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java
index f4f340fb6ace7..0bf64523f1239 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSelectKeyTest.java
@@ -16,20 +16,27 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
@@ -39,8 +46,27 @@ public class KStreamSelectKeyTest {
 
     final private Serde integerSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(topicName, new StringSerializer(), new IntegerSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-select-key-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testSelectKey() {
@@ -68,10 +94,10 @@ public String apply(Object key, Number value) {
 
         stream.selectKey(selector).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
 
         for (int expectedValue : expectedValues) {
-            driver.process(topicName, null, expectedValue);
+            driver.pipeInput(recordFactory.create(expectedValue));
         }
 
         assertEquals(3, processor.processed.size());
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java
index df3ceaf3257cc..aa0cf7ee7a5c6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java
@@ -16,20 +16,31 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Transformer;
 import org.apache.kafka.streams.kstream.TransformerSupplier;
 import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.PunctuationType;
+import org.apache.kafka.streams.processor.Punctuator;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.KStreamTestDriver;
 import org.apache.kafka.test.MockProcessorSupplier;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.util.Properties;
+
 import static org.junit.Assert.assertEquals;
 
 public class KStreamTransformTest {
@@ -37,8 +48,30 @@ public class KStreamTransformTest {
     private String topicName = "topic";
 
     final private Serde intSerde = Serdes.Integer();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
     @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    public final KStreamTestDriver kstreamDriver = new KStreamTestDriver();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-transform-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testTransform() {
@@ -79,13 +112,77 @@ public void close() {
         KStream stream = builder.stream(topicName, Consumed.with(intSerde, intSerde));
         stream.transform(transformerSupplier).process(processor);
 
-        driver.setUp(builder);
+        kstreamDriver.setUp(builder);
+        for (int expectedKey : expectedKeys) {
+            kstreamDriver.process(topicName, expectedKey, expectedKey * 10);
+        }
+
+        kstreamDriver.punctuate(2);
+        kstreamDriver.punctuate(3);
+
+        assertEquals(6, processor.processed.size());
+
+        String[] expected = {"2:10", "20:110", "200:1110", "2000:11110", "-1:2", "-1:3"};
+
+        for (int i = 0; i < expected.length; i++) {
+            assertEquals(expected[i], processor.processed.get(i));
+        }
+    }
+
+    @Test
+    public void testTransformWithNewDriverAndPunctuator() {
+        StreamsBuilder builder = new StreamsBuilder();
+
+        TransformerSupplier> transformerSupplier =
+            new TransformerSupplier>() {
+                public Transformer> get() {
+                    return new Transformer>() {
+
+                        private int total = 0;
+
+                        @Override
+                        public void init(final ProcessorContext context) {
+                            context.schedule(1, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {
+                                @Override
+                                public void punctuate(long timestamp) {
+                                    context.forward(-1, (int) timestamp);
+                                }
+                            });
+                        }
+
+                        @Override
+                        public KeyValue transform(Number key, Number value) {
+                            total += value.intValue();
+                            return KeyValue.pair(key.intValue() * 2, total);
+                        }
+
+                        @Override
+                        public KeyValue punctuate(long timestamp) {
+                            return null;
+                        }
+
+                        @Override
+                        public void close() {
+                        }
+                    };
+                }
+            };
+
+        final int[] expectedKeys = {1, 10, 100, 1000};
+
+        MockProcessorSupplier processor = new MockProcessorSupplier<>();
+        KStream stream = builder.stream(topicName, Consumed.with(intSerde, intSerde));
+        stream.transform(transformerSupplier).process(processor);
+
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, expectedKey * 10);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, 0L));
         }
 
-        driver.punctuate(2);
-        driver.punctuate(3);
+        // This tick will yield yields the "-1:2" result
+        driver.advanceWallClockTime(2);
+        // This tick further advances the clock to 3, which leads to the "-1:3" result
+        driver.advanceWallClockTime(1);
 
         assertEquals(6, processor.processed.size());
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
index dc0b886a93cf7..59a6a212f8945 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformValuesTest.java
@@ -16,10 +16,13 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.ValueTransformer;
@@ -29,11 +32,15 @@
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.To;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.junit.Rule;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Properties;
+
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.fail;
 
@@ -42,8 +49,27 @@ public class KStreamTransformValuesTest {
     private String topicName = "topic";
 
     final private Serde intSerde = Serdes.Integer();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new IntegerSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
+
+    @Before
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-transform-values-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
+    }
 
     @Test
     public void testTransform() {
@@ -85,9 +111,10 @@ public void close() {
         stream = builder.stream(topicName, Consumed.with(intSerde, intSerde));
         stream.transformValues(valueTransformerSupplier).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
+
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, expectedKey * 10);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, 0L));
         }
         String[] expected = {"1:10", "10:110", "100:1110", "1000:11110"};
 
@@ -128,9 +155,10 @@ public void close() {
         stream = builder.stream(topicName, Consumed.with(intSerde, intSerde));
         stream.transformValues(valueTransformerSupplier).process(processor);
 
-        driver.setUp(builder);
+        driver = new TopologyTestDriver(builder.build(), props);
+
         for (int expectedKey : expectedKeys) {
-            driver.process(topicName, expectedKey, expectedKey * 10);
+            driver.pipeInput(recordFactory.create(topicName, expectedKey, expectedKey * 10, 0L));
         }
         String[] expected = {"1:11", "10:121", "100:1221", "1000:12221"};
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
index 7082251888946..fc31db9167b1e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java
@@ -18,10 +18,13 @@
 
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
@@ -29,20 +32,18 @@
 import org.apache.kafka.streams.kstream.TimeWindows;
 import org.apache.kafka.streams.kstream.ValueJoiner;
 import org.apache.kafka.streams.kstream.Windowed;
-import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
 import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
 import org.apache.kafka.streams.state.WindowStore;
-import org.apache.kafka.test.InternalMockProcessorContext;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockAggregator;
 import org.apache.kafka.test.MockInitializer;
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
+import java.util.Properties;
 
 import static org.apache.kafka.test.StreamsTestUtils.getMetricByName;
 import static org.hamcrest.CoreMatchers.hasItem;
@@ -52,13 +53,26 @@
 public class KStreamWindowAggregateTest {
 
     final private Serde strSerde = Serdes.String();
-    private File stateDir = null;
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new StringSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
 
     @Before
-    public void setUp() {
-        stateDir = TestUtils.tempDirectory("kafka-test");
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "kstream-window-aggregate-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test
@@ -74,55 +88,24 @@ public void testAggBasic() {
         final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>();
         table2.toStream().process(proc2);
 
-        driver.setUp(builder, stateDir);
-
-        setRecordContext(0, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
-        setRecordContext(1, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(2, topic1);
-        driver.process(topic1, "C", "3");
-        driver.flushState();
-        setRecordContext(3, topic1);
-        driver.process(topic1, "D", "4");
-        driver.flushState();
-        setRecordContext(4, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
-
-        setRecordContext(5, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
-        setRecordContext(6, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(7, topic1);
-        driver.process(topic1, "D", "4");
-        driver.flushState();
-        setRecordContext(8, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(9, topic1);
-        driver.process(topic1, "C", "3");
-        driver.flushState();
-        setRecordContext(10, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
-        setRecordContext(11, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(12, topic1);
-        driver.flushState();
-        driver.process(topic1, "D", "4");
-        driver.flushState();
-        setRecordContext(13, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(14, topic1);
-        driver.process(topic1, "C", "3");
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
+
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 0L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 1L));
+        driver.pipeInput(recordFactory.create(topic1, "C", "3", 2L));
+        driver.pipeInput(recordFactory.create(topic1, "D", "4", 3L));
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 4L));
+
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 5L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 6L));
+        driver.pipeInput(recordFactory.create(topic1, "D", "4", 7L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 8L));
+        driver.pipeInput(recordFactory.create(topic1, "C", "3", 9L));
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 10L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 11L));
+        driver.pipeInput(recordFactory.create(topic1, "D", "4", 12L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 13L));
+        driver.pipeInput(recordFactory.create(topic1, "C", "3", 14L));
 
 
         assertEquals(
@@ -149,10 +132,6 @@ public void testAggBasic() {
         );
     }
 
-    private void setRecordContext(final long time, @SuppressWarnings("SameParameterValue") final String topic) {
-        ((InternalMockProcessorContext) driver.context()).setRecordContext(new ProcessorRecordContext(time, 0, 0, topic));
-    }
-
     @Test
     public void testJoin() {
         final StreamsBuilder builder = new StreamsBuilder();
@@ -183,23 +162,13 @@ public String apply(final String p1, final String p2) {
             }
         }).toStream().process(proc3);
 
-        driver.setUp(builder, stateDir);
-
-        setRecordContext(0, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
-        setRecordContext(1, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(2, topic1);
-        driver.process(topic1, "C", "3");
-        driver.flushState();
-        setRecordContext(3, topic1);
-        driver.process(topic1, "D", "4");
-        driver.flushState();
-        setRecordContext(4, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
+
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 0L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 1L));
+        driver.pipeInput(recordFactory.create(topic1, "C", "3", 2L));
+        driver.pipeInput(recordFactory.create(topic1, "D", "4", 3L));
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 4L));
 
         proc1.checkAndClearProcessResult(
             "[A@0/10]:0+1",
@@ -211,21 +180,11 @@ public String apply(final String p1, final String p2) {
         proc2.checkAndClearProcessResult();
         proc3.checkAndClearProcessResult();
 
-        setRecordContext(5, topic1);
-        driver.process(topic1, "A", "1");
-        driver.flushState();
-        setRecordContext(6, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(7, topic1);
-        driver.process(topic1, "D", "4");
-        driver.flushState();
-        setRecordContext(8, topic1);
-        driver.process(topic1, "B", "2");
-        driver.flushState();
-        setRecordContext(9, topic1);
-        driver.process(topic1, "C", "3");
-        driver.flushState();
+        driver.pipeInput(recordFactory.create(topic1, "A", "1", 5L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 6L));
+        driver.pipeInput(recordFactory.create(topic1, "D", "4", 7L));
+        driver.pipeInput(recordFactory.create(topic1, "B", "2", 8L));
+        driver.pipeInput(recordFactory.create(topic1, "C", "3", 9L));
 
         proc1.checkAndClearProcessResult(
             "[A@0/10]:0+1+1+1", "[A@5/15]:0+1",
@@ -237,21 +196,11 @@ public String apply(final String p1, final String p2) {
         proc2.checkAndClearProcessResult();
         proc3.checkAndClearProcessResult();
 
-        setRecordContext(0, topic1);
-        driver.process(topic2, "A", "a");
-        driver.flushState();
-        setRecordContext(1, topic1);
-        driver.process(topic2, "B", "b");
-        driver.flushState();
-        setRecordContext(2, topic1);
-        driver.process(topic2, "C", "c");
-        driver.flushState();
-        setRecordContext(3, topic1);
-        driver.process(topic2, "D", "d");
-        driver.flushState();
-        setRecordContext(4, topic1);
-        driver.process(topic2, "A", "a");
-        driver.flushState();
+        driver.pipeInput(recordFactory.create(topic2, "A", "a", 0L));
+        driver.pipeInput(recordFactory.create(topic2, "B", "b", 1L));
+        driver.pipeInput(recordFactory.create(topic2, "C", "c", 2L));
+        driver.pipeInput(recordFactory.create(topic2, "D", "d", 3L));
+        driver.pipeInput(recordFactory.create(topic2, "A", "a", 4L));
 
         proc1.checkAndClearProcessResult();
         proc2.checkAndClearProcessResult(
@@ -268,21 +217,12 @@ public String apply(final String p1, final String p2) {
             "[D@0/10]:0+4+4%0+d",
             "[A@0/10]:0+1+1+1%0+a+a");
 
-        setRecordContext(5, topic1);
-        driver.process(topic2, "A", "a");
-        driver.flushState();
-        setRecordContext(6, topic1);
-        driver.process(topic2, "B", "b");
-        driver.flushState();
-        setRecordContext(7, topic1);
-        driver.process(topic2, "D", "d");
-        driver.flushState();
-        setRecordContext(8, topic1);
-        driver.process(topic2, "B", "b");
-        driver.flushState();
-        setRecordContext(9, topic1);
-        driver.process(topic2, "C", "c");
-        driver.flushState();
+        driver.pipeInput(recordFactory.create(topic2, "A", "a", 5L));
+        driver.pipeInput(recordFactory.create(topic2, "B", "b", 6L));
+        driver.pipeInput(recordFactory.create(topic2, "D", "d", 7L));
+        driver.pipeInput(recordFactory.create(topic2, "B", "b", 8L));
+        driver.pipeInput(recordFactory.create(topic2, "C", "c", 9L));
+
         proc1.checkAndClearProcessResult();
         proc2.checkAndClearProcessResult(
             "[A@0/10]:0+a+a+a", "[A@5/15]:0+a",
@@ -314,15 +254,13 @@ public void shouldLogAndMeterWhenSkippingNullKey() {
                 Materialized.>as("topic1-Canonicalized").withValueSerde(strSerde)
             );
 
-        driver.setUp(builder, stateDir);
+        driver = new TopologyTestDriver(builder.build(), props, 0L);
 
-        setRecordContext(0, topic);
         final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();
-        driver.process(topic, null, "1");
-        driver.flushState();
+        driver.pipeInput(recordFactory.create(topic, null, "1"));
         LogCaptureAppender.unregister(appender);
 
-        assertEquals(1.0, getMetricByName(driver.context().metrics().metrics(), "skipped-records-total", "stream-metrics").metricValue());
-        assertThat(appender.getMessages(), hasItem("Skipping record due to null key. value=[1] topic=[topic] partition=[-1] offset=[-1]"));
+        assertEquals(1.0, getMetricByName(driver.metrics(), "skipped-records-total", "stream-metrics").metricValue());
+        assertThat(appender.getMessages(), hasItem("Skipping record due to null key. value=[1] topic=[topic] partition=[0] offset=[0]"));
     }
 }
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableForeachTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableForeachTest.java
index d8b3a5f382353..30c0a7a51c3e8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableForeachTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableForeachTest.java
@@ -16,27 +16,31 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.Consumed;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriver;
 import org.apache.kafka.streams.kstream.ForeachAction;
 import org.apache.kafka.streams.kstream.KTable;
 import org.apache.kafka.streams.kstream.Materialized;
 import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.TestUtils;
+import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
@@ -44,15 +48,28 @@
 public class KTableForeachTest {
 
     final private String topicName = "topic";
-    private File stateDir = null;
     final private Serde intSerde = Serdes.Integer();
     final private Serde stringSerde = Serdes.String();
-    @Rule
-    public final KStreamTestDriver driver = new KStreamTestDriver();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(new IntegerSerializer(), new StringSerializer());
+    private TopologyTestDriver driver;
+    private final Properties props = new Properties();
 
     @Before
-    public void setUp() {
-        stateDir = TestUtils.tempDirectory("kafka-test");
+    public void setup() {
+        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "ktable-foreach-test");
+        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
+        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
+        props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass().getName());
+        props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        props.clear();
+        if (driver != null) {
+            driver.close();
+        }
+        driver = null;
     }
 
     @Test
@@ -91,11 +108,11 @@ public void apply(Integer key, String value) {
         table.foreach(action);
 
         // Then
-        driver.setUp(builder, stateDir);
+        driver = new TopologyTestDriver(builder.build(), props);
+
         for (KeyValue record: inputRecords) {
-            driver.process(topicName, record.key, record.value);
+            driver.pipeInput(recordFactory.create(topicName, record.key, record.value));
         }
-        driver.flushState();
 
         assertEquals(expectedRecords.size(), actualRecords.size());
         for (int i = 0; i < expectedRecords.size(); i++) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
index f67b6341f8d8e..f948c2c7faebc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
@@ -16,9 +16,11 @@
  */
 package org.apache.kafka.streams.processor;
 
+import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.config.TopicConfig;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.TopologyTestDriverWrapper;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.errors.StreamsException;
 import org.apache.kafka.streams.errors.TopologyBuilderException;
@@ -34,7 +36,6 @@
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockStateStoreSupplier;
 import org.apache.kafka.test.MockTimestampExtractor;
-import org.apache.kafka.test.ProcessorTopologyTestDriver;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
@@ -605,16 +606,12 @@ public void shouldThroughOnUnassignedStateStoreAccess() throws Exception {
 
         final TopologyBuilder builder = new TopologyBuilder();
         builder.addSource(sourceNodeName, "topic")
-                .addProcessor(goodNodeName, new LocalMockProcessorSupplier(),
-                        sourceNodeName)
-                .addStateStore(
-                        Stores.create(LocalMockProcessorSupplier.STORE_NAME)
-                                .withStringKeys().withStringValues().inMemory()
-                                .build(), goodNodeName)
-                .addProcessor(badNodeName, new LocalMockProcessorSupplier(),
-                        sourceNodeName);     
+                .addProcessor(goodNodeName, new LocalMockProcessorSupplier(), sourceNodeName)
+                .addStateStore(Stores.create(LocalMockProcessorSupplier.STORE_NAME).withStringKeys().withStringValues().inMemory().build(), goodNodeName)
+                .addProcessor(badNodeName, new LocalMockProcessorSupplier(), sourceNodeName);
         try {
-            final ProcessorTopologyTestDriver driver = new ProcessorTopologyTestDriver(streamsConfig, builder.internalTopologyBuilder);
+            final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(builder.internalTopologyBuilder, config);
+            driver.pipeInput(new ConsumerRecord<>("topic", 0, 0L, new byte[] {}, new byte[] {}));
             fail("Should have thrown StreamsException");
         } catch (final StreamsException e) {
             final String error = e.toString();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java
index 901fc4b7e0aef..25468c0d449ce 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java
@@ -19,6 +19,7 @@
 import org.apache.kafka.common.config.TopicConfig;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.TopologyTestDriverWrapper;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.TopologyDescription;
@@ -34,7 +35,6 @@
 import org.apache.kafka.test.MockProcessorSupplier;
 import org.apache.kafka.test.MockStateStoreSupplier;
 import org.apache.kafka.test.MockTimestampExtractor;
-import org.apache.kafka.test.ProcessorTopologyTestDriver;
 import org.apache.kafka.test.TestUtils;
 import org.junit.Test;
 
@@ -581,7 +581,7 @@ public void shouldThrowOnUnassignedStateStoreAccess() {
         builder.addProcessor(badNodeName, new LocalMockProcessorSupplier(), sourceNodeName);
         
         try {
-            new ProcessorTopologyTestDriver(streamsConfig, builder);
+            new TopologyTestDriverWrapper(builder, config);
             fail("Should have throw StreamsException");
         } catch (final StreamsException e) {
             final String error = e.toString();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java
index d07274a32a92e..a80b25d027151 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java
@@ -24,6 +24,7 @@
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.TopologyTestDriverWrapper;
 import org.apache.kafka.streams.processor.AbstractProcessor;
 import org.apache.kafka.streams.processor.Processor;
 import org.apache.kafka.streams.processor.ProcessorContext;
@@ -36,8 +37,8 @@
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.Stores;
+import org.apache.kafka.streams.test.ConsumerRecordFactory;
 import org.apache.kafka.test.MockProcessorSupplier;
-import org.apache.kafka.test.ProcessorTopologyTestDriver;
 import org.apache.kafka.test.TestUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -66,26 +67,26 @@ public class ProcessorTopologyTest {
 
     private final TopologyBuilder builder = new TopologyBuilder();
     private final MockProcessorSupplier mockProcessorSupplier = new MockProcessorSupplier();
+    private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>(STRING_SERIALIZER, STRING_SERIALIZER, 0L);
 
-    private ProcessorTopologyTestDriver driver;
-    private StreamsConfig config;
+    private TopologyTestDriverWrapper driver;
+    private final Properties props = new Properties();
 
     @Before
     public void setup() {
         // Create a new directory in which we'll put all of the state for this test, enabling running tests in parallel ...
         final File localState = TestUtils.tempDirectory();
-        final Properties props = new Properties();
         props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "processor-topology-test");
         props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
         props.setProperty(StreamsConfig.STATE_DIR_CONFIG, localState.getAbsolutePath());
         props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
         props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
         props.setProperty(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, CustomTimestampExtractor.class.getName());
-        this.config = new StreamsConfig(props);
     }
 
     @After
     public void cleanup() {
+        props.clear();
         if (driver != null) {
             driver.close();
         }
@@ -122,19 +123,19 @@ public void testTopologyMetadata() {
 
     @Test
     public void testDrivingSimpleTopology() {
-        final int partition = 10;
-        driver = new ProcessorTopologyTestDriver(config, createSimpleTopology(partition).internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
+        int partition = 10;
+        driver = new TopologyTestDriverWrapper(createSimpleTopology(partition).internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition);
         assertNoOutputRecord(OUTPUT_TOPIC_2);
 
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition);
         assertNoOutputRecord(OUTPUT_TOPIC_2);
 
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key4", "value4", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key5", "value5", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key4", "value4"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key5", "value5"));
         assertNoOutputRecord(OUTPUT_TOPIC_2);
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition);
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key4", "value4", partition);
@@ -144,18 +145,18 @@ public void testDrivingSimpleTopology() {
 
     @Test
     public void testDrivingMultiplexingTopology() {
-        driver = new ProcessorTopologyTestDriver(config, createMultiplexingTopology().internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver = new TopologyTestDriverWrapper(createMultiplexingTopology().internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1(2)");
 
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2(2)");
 
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key4", "value4", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key5", "value5", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key4", "value4"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key5", "value5"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key4", "value4(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key5", "value5(1)");
@@ -166,18 +167,18 @@ public void testDrivingMultiplexingTopology() {
 
     @Test
     public void testDrivingMultiplexByNameTopology() {
-        driver = new ProcessorTopologyTestDriver(config, createMultiplexByNameTopology().internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver = new TopologyTestDriverWrapper(createMultiplexByNameTopology().internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1(2)");
 
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2(2)");
 
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key4", "value4", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key5", "value5", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key4", "value4"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key5", "value5"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key4", "value4(1)");
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key5", "value5(1)");
@@ -188,12 +189,12 @@ public void testDrivingMultiplexByNameTopology() {
 
     @Test
     public void testDrivingStatefulTopology() {
-        final String storeName = "entries";
-        driver = new ProcessorTopologyTestDriver(config, createStatefulTopology(storeName).internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key1", "value4", STRING_SERIALIZER, STRING_SERIALIZER);
+        String storeName = "entries";
+        driver = new TopologyTestDriverWrapper(createStatefulTopology(storeName).internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value4"));
         assertNoOutputRecord(OUTPUT_TOPIC_1);
 
         final KeyValueStore store = driver.getKeyValueStore(storeName);
@@ -214,10 +215,10 @@ public void shouldDriveGlobalStore() {
         final TopologyBuilder topologyBuilder = this.builder
                 .addGlobalStore(storeSupplier, global, STRING_DESERIALIZER, STRING_DESERIALIZER, topic, "processor", define(new StatefulProcessor(storeName)));
 
-        driver = new ProcessorTopologyTestDriver(config, topologyBuilder.internalTopologyBuilder);
-        final KeyValueStore globalStore = (KeyValueStore) topologyBuilder.globalStateStores().get(storeName);
-        driver.process(topic, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(topic, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver = new TopologyTestDriverWrapper(topologyBuilder.internalTopologyBuilder, props);
+        final KeyValueStore globalStore = (KeyValueStore) topologyBuilder.globalStateStores().get("my-store");
+        driver.pipeInput(recordFactory.create(topic, "key1", "value1"));
+        driver.pipeInput(recordFactory.create(topic, "key2", "value2"));
         assertEquals("value1", globalStore.get("key1"));
         assertEquals("value2", globalStore.get("key2"));
     }
@@ -225,23 +226,23 @@ public void shouldDriveGlobalStore() {
     @Test
     public void testDrivingSimpleMultiSourceTopology() {
         final int partition = 10;
-        driver = new ProcessorTopologyTestDriver(config, createSimpleMultiSourceTopology(partition).internalTopologyBuilder);
+        driver = new TopologyTestDriverWrapper(createSimpleMultiSourceTopology(partition).internalTopologyBuilder, props);
 
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition);
         assertNoOutputRecord(OUTPUT_TOPIC_2);
 
-        driver.process(INPUT_TOPIC_2, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_2, "key2", "value2"));
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2", partition);
         assertNoOutputRecord(OUTPUT_TOPIC_1);
     }
 
     @Test
     public void testDrivingForwardToSourceTopology() {
-        driver = new ProcessorTopologyTestDriver(config, createForwardToSourceTopology().internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver = new TopologyTestDriverWrapper(createForwardToSourceTopology().internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key1", "value1");
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key2", "value2");
         assertNextOutputRecord(OUTPUT_TOPIC_2, "key3", "value3");
@@ -249,10 +250,10 @@ public void testDrivingForwardToSourceTopology() {
 
     @Test
     public void testDrivingInternalRepartitioningTopology() {
-        driver = new ProcessorTopologyTestDriver(config, createInternalRepartitioningTopology().internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver = new TopologyTestDriverWrapper(createInternalRepartitioningTopology().internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3"));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1");
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2");
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3");
@@ -260,10 +261,10 @@ public void testDrivingInternalRepartitioningTopology() {
 
     @Test
     public void testDrivingInternalRepartitioningForwardingTimestampTopology() {
-        driver = new ProcessorTopologyTestDriver(config, createInternalRepartitioningWithValueTimestampTopology().internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1@1000", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key2", "value2@2000", STRING_SERIALIZER, STRING_SERIALIZER);
-        driver.process(INPUT_TOPIC_1, "key3", "value3@3000", STRING_SERIALIZER, STRING_SERIALIZER);
+        driver = new TopologyTestDriverWrapper(createInternalRepartitioningWithValueTimestampTopology().internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1@1000"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2@2000"));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3@3000"));
         assertThat(driver.readOutput(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER),
                 equalTo(new ProducerRecord<>(OUTPUT_TOPIC_1, null, 1000L, "key1", "value1")));
         assertThat(driver.readOutput(OUTPUT_TOPIC_1, STRING_DESERIALIZER, STRING_DESERIALIZER),
@@ -319,10 +320,10 @@ public void shouldRecursivelyPrintChildren() {
     @Test
     public void shouldConsiderTimeStamps() {
         final int partition = 10;
-        driver = new ProcessorTopologyTestDriver(config, createSimpleTopology(partition).internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER, 10L);
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER, 20L);
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER, 30L);
+        driver = new TopologyTestDriverWrapper(createSimpleTopology(partition).internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3", 30L));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 10L);
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 20L);
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition, 30L);
@@ -331,10 +332,10 @@ public void shouldConsiderTimeStamps() {
     @Test
     public void shouldConsiderModifiedTimeStamps() {
         final int partition = 10;
-        driver = new ProcessorTopologyTestDriver(config, createTimestampTopology(partition).internalTopologyBuilder);
-        driver.process(INPUT_TOPIC_1, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER, 10L);
-        driver.process(INPUT_TOPIC_1, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER, 20L);
-        driver.process(INPUT_TOPIC_1, "key3", "value3", STRING_SERIALIZER, STRING_SERIALIZER, 30L);
+        driver = new TopologyTestDriverWrapper(createTimestampTopology(partition).internalTopologyBuilder, props);
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key1", "value1", 10L));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key2", "value2", 20L));
+        driver.pipeInput(recordFactory.create(INPUT_TOPIC_1, "key3", "value3", 30L));
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key1", "value1", partition, 20L);
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key2", "value2", partition, 30L);
         assertNextOutputRecord(OUTPUT_TOPIC_1, "key3", "value3", partition, 40L);
diff --git a/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java b/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java
index 3daf051186762..c93a30636a090 100644
--- a/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java
+++ b/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java
@@ -44,6 +44,7 @@
 import java.util.Set;
 import java.util.regex.Pattern;
 
+@Deprecated
 public class KStreamTestDriver extends ExternalResource {
 
     private static final long DEFAULT_CACHE_SIZE_BYTES = 1024 * 1024L;
diff --git a/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java b/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java
deleted file mode 100644
index afd2bb20d723f..0000000000000
--- a/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java
+++ /dev/null
@@ -1,490 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.test;
-
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.MockConsumer;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.clients.producer.MockProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.metrics.Metrics;
-import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.MockTime;
-import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.streams.StreamsConfig;
-import org.apache.kafka.streams.Topology;
-import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler;
-import org.apache.kafka.streams.processor.StateStore;
-import org.apache.kafka.streams.processor.TaskId;
-import org.apache.kafka.streams.processor.internals.GlobalProcessorContextImpl;
-import org.apache.kafka.streams.processor.internals.GlobalStateManagerImpl;
-import org.apache.kafka.streams.processor.internals.GlobalStateUpdateTask;
-import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
-import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
-import org.apache.kafka.streams.processor.internals.MockStreamsMetrics;
-import org.apache.kafka.streams.processor.internals.ProcessorContextImpl;
-import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
-import org.apache.kafka.streams.processor.internals.ProcessorTopology;
-import org.apache.kafka.streams.processor.internals.StateDirectory;
-import org.apache.kafka.streams.processor.internals.StoreChangelogReader;
-import org.apache.kafka.streams.processor.internals.StreamTask;
-import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
-import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.state.internals.ThreadCache;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class makes it easier to write tests to verify the behavior of topologies created with a {@link Topology}.
- * You can test simple topologies that have a single processor, or very complex topologies that have multiple sources, processors,
- * and sinks. And because it starts with a {@link Topology}, you can create topologies specific to your tests or you
- * can use and test code you already have that uses a builder to create topologies. Best of all, the class works without a real
- * Kafka broker, so the tests execute very quickly with very little overhead.
- * 
    
- * Using the ProcessorTopologyTestDriver in tests is easy: simply instantiate the driver with a {@link StreamsConfig} and a
- * TopologyBuilder, use the driver to supply an input message to the topology, and then use the driver to read and verify any
- * messages output by the topology.
- * 
    
- * Although the driver doesn't use a real Kafka broker, it does simulate Kafka {@link org.apache.kafka.clients.consumer.Consumer}s
- * and {@link org.apache.kafka.clients.producer.Producer}s that read and write raw {@code byte[]} messages. You can either deal
- * with messages that have {@code byte[]} keys and values, or you can supply the {@link Serializer}s and {@link Deserializer}s
- * that the driver can use to convert the keys and values into objects.
- *
- * 
    Driver setup
    
- * 
    
- * In order to create a ProcessorTopologyTestDriver instance, you need a TopologyBuilder and a {@link StreamsConfig}. The
- * configuration needs to be representative of what you'd supply to the real topology, so that means including several key
- * properties. For example, the following code fragment creates a configuration that specifies a local Kafka broker list
- * (which is needed but not used), a timestamp extractor, and default serializers and deserializers for string keys and values:
- *
- * 
    - * StringSerializer strSerializer = new StringSerializer();
- * StringDeserializer strDeserializer = new StringDeserializer();
- * Properties props = new Properties();
- * props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091");
- * props.setProperty(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, CustomTimestampExtractor.class.getName());
- * props.setProperty(StreamsConfig.KEY_SERIALIZER_CLASS_CONFIG, strSerializer.getClass().getName());
- * props.setProperty(StreamsConfig.KEY_DESERIALIZER_CLASS_CONFIG, strDeserializer.getClass().getName());
- * props.setProperty(StreamsConfig.VALUE_SERIALIZER_CLASS_CONFIG, strSerializer.getClass().getName());
- * props.setProperty(StreamsConfig.VALUE_DESERIALIZER_CLASS_CONFIG, strDeserializer.getClass().getName());
- * StreamsConfig config = new StreamsConfig(props);
- * TopologyBuilder builder = ...
- * ProcessorTopologyTestDriver driver = new ProcessorTopologyTestDriver(config, builder);
- * 
    
- *
- * 
    Processing messages
    
- * 
    
- * Your test can supply new input records on any of the topics that the topology's sources consume. Here's an example of an
- * input message on the topic named {@code input-topic}:
- *
- * 
    - * driver.process("input-topic", "key1", "value1", strSerializer, strSerializer);
- * 
    
- *
- * Immediately, the driver will pass the input message through to the appropriate source that consumes the named topic,
- * and will invoke the processor(s) downstream of the source. If your topology's processors forward messages to sinks,
- * your test can then consume these output messages to verify they match the expected outcome. For example, if our topology
- * should have generated 2 messages on {@code output-topic-1} and 1 message on {@code output-topic-2}, then our test can
- * obtain these messages using the {@link #readOutput(String, Deserializer, Deserializer)} method:
- *
- * 
    - * ProducerRecord record1 = driver.readOutput("output-topic-1", strDeserializer, strDeserializer);
- * ProducerRecord record2 = driver.readOutput("output-topic-1", strDeserializer, strDeserializer);
- * ProducerRecord record3 = driver.readOutput("output-topic-2", strDeserializer, strDeserializer);
- * 
    
- *
- * Again, our example topology generates messages with string keys and values, so we supply our string deserializer instance
- * for use on both the keys and values. Your test logic can then verify whether these output records are correct.
- * 
    
- * Finally, when completed, make sure your tests {@link #close()} the driver to release all resources and
- * {@link org.apache.kafka.streams.processor.Processor}s.
- *
- * 
    Processor state
    
- * 
    
- * Some processors use Kafka {@link StateStore state storage}, so this driver class provides the {@link #getStateStore(String)}
- * and {@link #getKeyValueStore(String)} methods so that your tests can check the underlying state store(s) used by your
- * topology's processors. In our previous example, after we supplied a single input message and checked the three output messages,
- * our test could also check the key value store to verify the processor correctly added, removed, or updated internal state.
- * Or, our test might have pre-populated some state before submitting the input message, and verified afterward that the
- * processor(s) correctly updated the state.
- */
-public class ProcessorTopologyTestDriver {
-
-    private final static String APPLICATION_ID = "test-driver-application";
-    private final static int PARTITION_ID = 0;
-    private final static TaskId TASK_ID = new TaskId(0, PARTITION_ID);
-
-    private final ProcessorTopology topology;
-    private final MockProducer producer;
-    private final Map partitionsByTopic = new HashMap<>();
-    private final Map offsetsByTopicPartition = new HashMap<>();
-    private final Map>> outputRecordsByTopic = new HashMap<>();
-    private final Set internalTopics = new HashSet<>();
-    private final Map globalPartitionsByTopic = new HashMap<>();
-    private StreamTask task;
-    private GlobalStateUpdateTask globalStateTask;
-
-    /**
-     * Create a new test driver instance.
-     *
-     * @param config  the stream configuration for the topology
-     * @param builder the topology builder that will be used to create the topology instance
-     */
-    public ProcessorTopologyTestDriver(final StreamsConfig config,
-                                       final InternalTopologyBuilder builder) {
-        topology = builder.setApplicationId(APPLICATION_ID).build(null);
-        final ProcessorTopology globalTopology = builder.buildGlobalStateTopology();
-
-        // Set up the consumer and producer ...
-        final Consumer consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
-        final Serializer bytesSerializer = new ByteArraySerializer();
-        producer = new MockProducer(true, bytesSerializer, bytesSerializer) {
-            @Override
-            public List partitionsFor(final String topic) {
-                return Collections.singletonList(new PartitionInfo(topic, PARTITION_ID, null, null, null));
-            }
-        };
-
-        // Identify internal topics for forwarding in process ...
-        for (final InternalTopologyBuilder.TopicsInfo topicsInfo : builder.topicGroups().values()) {
-            internalTopics.addAll(topicsInfo.repartitionSourceTopics.keySet());
-        }
-
-        // Set up all of the topic+partition information and subscribe the consumer to each ...
-        for (final String topic : topology.sourceTopics()) {
-            final TopicPartition tp = new TopicPartition(topic, PARTITION_ID);
-            partitionsByTopic.put(topic, tp);
-            offsetsByTopicPartition.put(tp, new AtomicLong());
-        }
-
-        consumer.assign(offsetsByTopicPartition.keySet());
-
-        final StateDirectory stateDirectory = new StateDirectory(config, Time.SYSTEM);
-        final StreamsMetricsImpl streamsMetrics = new MockStreamsMetrics(new Metrics());
-        final ThreadCache cache = new ThreadCache(new LogContext("mock "), 1024 * 1024, streamsMetrics);
-
-        if (globalTopology != null) {
-            final MockConsumer globalConsumer = createGlobalConsumer();
-            final MockStateRestoreListener stateRestoreListener = new MockStateRestoreListener();
-            for (final String topicName : globalTopology.sourceTopics()) {
-                final List partitionInfos = new ArrayList<>();
-                partitionInfos.add(new PartitionInfo(topicName, 1, null, null, null));
-                globalConsumer.updatePartitions(topicName, partitionInfos);
-                final TopicPartition partition = new TopicPartition(topicName, 1);
-                globalConsumer.updateEndOffsets(Collections.singletonMap(partition, 0L));
-                globalPartitionsByTopic.put(topicName, partition);
-                offsetsByTopicPartition.put(partition, new AtomicLong());
-            }
-            final GlobalStateManagerImpl stateManager = new GlobalStateManagerImpl(
-                new LogContext("mock "),
-                globalTopology,
-                globalConsumer,
-                stateDirectory,
-                stateRestoreListener,
-                config);
-            final GlobalProcessorContextImpl globalProcessorContext = new GlobalProcessorContextImpl(config, stateManager, streamsMetrics, cache);
-            stateManager.setGlobalProcessorContext(globalProcessorContext);
-            globalStateTask = new GlobalStateUpdateTask(
-                globalTopology,
-                globalProcessorContext,
-                stateManager,
-                new LogAndContinueExceptionHandler(),
-                new LogContext());
-            globalStateTask.initialize();
-        }
-
-        if (!partitionsByTopic.isEmpty()) {
-            task = new StreamTask(
-                TASK_ID,
-                partitionsByTopic.values(),
-                topology,
-                consumer,
-                new StoreChangelogReader(
-                    createRestoreConsumer(topology.storeToChangelogTopic()),
-                    new MockStateRestoreListener(),
-                    new LogContext("topology-test-driver ")
-                ),
-                config,
-                streamsMetrics,
-                stateDirectory,
-                cache,
-                new MockTime(),
-                producer
-            );
-            task.initializeStateStores();
-            task.initializeTopology();
-        }
-    }
-
-    /**
-     * Send an input message with the given key, value and timestamp on the specified topic to the topology, and then commit the messages.
-     *
-     * @param topicName the name of the topic on which the message is to be sent
-     * @param key       the raw message key
-     * @param value     the raw message value
-     * @param timestamp the raw message timestamp
-     */
-    public void process(final String topicName,
-                        final byte[] key,
-                        final byte[] value,
-                        final long timestamp) {
-
-        final TopicPartition tp = partitionsByTopic.get(topicName);
-        if (tp != null) {
-            // Add the record ...
-            final long offset = offsetsByTopicPartition.get(tp).incrementAndGet();
-            task.addRecords(tp, records(new ConsumerRecord<>(tp.topic(), tp.partition(), offset, timestamp, TimestampType.CREATE_TIME, 0L, 0, 0, key, value)));
-            producer.clear();
-
-            // Process the record ...
-            task.process();
-            ((InternalProcessorContext) task.context()).setRecordContext(new ProcessorRecordContext(timestamp, offset, tp.partition(), topicName));
-            task.commit();
-
-            // Capture all the records sent to the producer ...
-            for (final ProducerRecord record : producer.history()) {
-                Queue> outputRecords = outputRecordsByTopic.get(record.topic());
-                if (outputRecords == null) {
-                    outputRecords = new LinkedList<>();
-                    outputRecordsByTopic.put(record.topic(), outputRecords);
-                }
-                outputRecords.add(record);
-
-                // Forward back into the topology if the produced record is to an internal or a source topic ...
-                if (internalTopics.contains(record.topic()) || topology.sourceTopics().contains(record.topic())) {
-                    process(record.topic(), record.key(), record.value(), record.timestamp());
-                }
-            }
-        } else {
-            final TopicPartition global = globalPartitionsByTopic.get(topicName);
-            if (global == null) {
-                throw new IllegalArgumentException("Unexpected topic: " + topicName);
-            }
-            final long offset = offsetsByTopicPartition.get(global).incrementAndGet();
-            globalStateTask.update(new ConsumerRecord<>(global.topic(), global.partition(), offset, timestamp, TimestampType.CREATE_TIME, 0L, 0, 0, key, value));
-            globalStateTask.flushState();
-        }
-    }
-
-    /**
-     * Send an input message with the given key and value on the specified topic to the topology.
-     *
-     * @param topicName the name of the topic on which the message is to be sent
-     * @param key       the raw message key
-     * @param value     the raw message value
-     */
-    public void process(final String topicName,
-                        final byte[] key,
-                        final byte[] value) {
-        process(topicName, key, value, 0L);
-    }
-
-    /**
-     * Send an input message with the given key and value on the specified topic to the topology.
-     *
-     * @param topicName       the name of the topic on which the message is to be sent
-     * @param key             the raw message key
-     * @param value           the raw message value
-     * @param keySerializer   the serializer for the key
-     * @param valueSerializer the serializer for the value
-     */
-    public  void process(final String topicName,
-                               final K key,
-                               final V value,
-                               final Serializer keySerializer,
-                               final Serializer valueSerializer) {
-        process(topicName, key, value, keySerializer, valueSerializer, 0L);
-    }
-
-    /**
-     * Send an input message with the given key and value and timestamp on the specified topic to the topology.
-     *
-     * @param topicName       the name of the topic on which the message is to be sent
-     * @param key             the raw message key
-     * @param value           the raw message value
-     * @param keySerializer   the serializer for the key
-     * @param valueSerializer the serializer for the value
-     * @param timestamp       the raw message timestamp
-     */
-    public  void process(final String topicName,
-                               final K key,
-                               final V value,
-                               final Serializer keySerializer,
-                               final Serializer valueSerializer,
-                               final long timestamp) {
-        process(topicName, keySerializer.serialize(topicName, key), valueSerializer.serialize(topicName, value), timestamp);
-    }
-
-    /**
-     * Read the next record from the given topic. These records were output by the topology during the previous calls to
-     * {@link #process(String, byte[], byte[])}.
-     *
-     * @param topic the name of the topic
-     * @return the next record on that topic, or null if there is no record available
-     */
-    public ProducerRecord readOutput(final String topic) {
-        final Queue> outputRecords = outputRecordsByTopic.get(topic);
-        if (outputRecords == null) {
-            return null;
-        }
-        return outputRecords.poll();
-    }
-
-    /**
-     * Read the next record from the given topic. These records were output by the topology during the previous calls to
-     * {@link #process(String, byte[], byte[])}.
-     *
-     * @param topic             the name of the topic
-     * @param keyDeserializer   the deserializer for the key type
-     * @param valueDeserializer the deserializer for the value type
-     * @return the next record on that topic, or null if there is no record available
-     */
-    public  ProducerRecord readOutput(final String topic,
-                                                  final Deserializer keyDeserializer,
-                                                  final Deserializer valueDeserializer) {
-        final ProducerRecord record = readOutput(topic);
-        if (record == null) {
-            return null;
-        }
-        final K key = keyDeserializer.deserialize(record.topic(), record.key());
-        final V value = valueDeserializer.deserialize(record.topic(), record.value());
-        return new ProducerRecord<>(record.topic(), record.partition(), record.timestamp(), key, value);
-    }
-
-    private Iterable> records(final ConsumerRecord record) {
-        return Collections.singleton(record);
-    }
-
-    /**
-     * Get the {@link StateStore} with the given name. The name should have been supplied via
-     * {@link #ProcessorTopologyTestDriver(StreamsConfig, InternalTopologyBuilder) this object's constructor}, and is
-     * presumed to be used by a Processor within the topology.
-     * 
    
-     * This is often useful in test cases to pre-populate the store before the test case instructs the topology to
-     * {@link #process(String, byte[], byte[]) process an input message}, and/or to check the store afterward.
-     *
-     * @param name the name of the store
-     * @return the state store, or null if no store has been registered with the given name
-     * @see #getKeyValueStore(String)
-     */
-    public StateStore getStateStore(final String name) {
-        return ((ProcessorContextImpl) task.context()).getStateMgr().getStore(name);
-    }
-
-    /**
-     * Get the {@link KeyValueStore} with the given name. The name should have been supplied via
-     * {@link #ProcessorTopologyTestDriver(StreamsConfig, InternalTopologyBuilder) this object's constructor}, and is
-     * presumed to be used by a Processor within the topology.
-     * 
    
-     * This is often useful in test cases to pre-populate the store before the test case instructs the topology to
-     * {@link #process(String, byte[], byte[]) process an input message}, and/or to check the store afterward.
-     * 
    
-     *
-     * @param name the name of the store
-     * @return the key value store, or null if no {@link KeyValueStore} has been registered with the given name
-     * @see #getStateStore(String)
-     */
-    @SuppressWarnings("unchecked")
-    public  KeyValueStore getKeyValueStore(final String name) {
-        final StateStore store = getStateStore(name);
-        return store instanceof KeyValueStore ? (KeyValueStore) getStateStore(name) : null;
-    }
-
-    /**
-     * Close the driver, its topology, and all processors.
-     */
-    public void close() {
-        if (task != null) {
-            task.close(true, false);
-        }
-        if (globalStateTask != null) {
-            try {
-                globalStateTask.close();
-            } catch (final IOException e) {
-                // ignore
-            }
-        }
-    }
-
-    /**
-     * Utility method that creates the {@link MockConsumer} used for restoring state, which should not be done by this
-     * driver object unless this method is overwritten with a functional consumer.
-     *
-     * @param storeToChangelogTopic the map of the names of the stores to the changelog topics
-     * @return the mock consumer; never null
-     */
-    private MockConsumer createRestoreConsumer(final Map storeToChangelogTopic) {
-        final MockConsumer consumer = new MockConsumer(OffsetResetStrategy.LATEST) {
-            @Override
-            public synchronized void seekToEnd(final Collection partitions) {}
-
-            @Override
-            public synchronized void seekToBeginning(final Collection partitions) {}
-
-            @Override
-            public synchronized long position(final TopicPartition partition) {
-                return 0L;
-            }
-        };
-        // For each store ...
-        for (final Map.Entry storeAndTopic : storeToChangelogTopic.entrySet()) {
-            final String topicName = storeAndTopic.getValue();
-            // Set up the restore-state topic ...
-            // consumer.subscribe(new TopicPartition(topicName, 1));
-            // Set up the partition that matches the ID (which is what ProcessorStateManager expects) ...
-            final List partitionInfos = new ArrayList<>();
-            partitionInfos.add(new PartitionInfo(topicName, PARTITION_ID, null, null, null));
-            consumer.updatePartitions(topicName, partitionInfos);
-            consumer.updateEndOffsets(Collections.singletonMap(new TopicPartition(topicName, PARTITION_ID), 0L));
-        }
-        return consumer;
-    }
-
-    private MockConsumer createGlobalConsumer() {
-        return new MockConsumer(OffsetResetStrategy.LATEST) {
-            @Override
-            public synchronized void seekToEnd(final Collection partitions) {}
-
-            @Override
-            public synchronized void seekToBeginning(final Collection partitions) {}
-
-            @Override
-            public synchronized long position(final TopicPartition partition) {
-                return 0L;
-            }
-        };
-    }
-
-}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index bde70b4c6d499..7343d59e01381 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -216,10 +216,36 @@ public TopologyTestDriver(final Topology topology,
     public TopologyTestDriver(final Topology topology,
                               final Properties config,
                               final long initialWallClockTimeMs) {
+
+        this(topology.internalTopologyBuilder, config, initialWallClockTimeMs);
+    }
+
+    /**
+     * Create a new test diver instance.
+     *
+     * @param builder builder for the topology to be tested
+     * @param config the configuration for the topology
+     */
+    protected TopologyTestDriver(final InternalTopologyBuilder builder,
+                              final Properties config) {
+        this(builder, config,  System.currentTimeMillis());
+
+    }
+
+    /**
+     * Create a new test diver instance.
+     *
+     * @param builder builder for the topology to be tested
+     * @param config the configuration for the topology
+     * @param initialWallClockTimeMs the initial value of internally mocked wall-clock time
+     */
+    private TopologyTestDriver(final InternalTopologyBuilder builder,
+                              final Properties config,
+                              final long initialWallClockTimeMs) {
         final StreamsConfig streamsConfig = new StreamsConfig(config);
         mockTime = new MockTime(initialWallClockTimeMs);
 
-        internalTopologyBuilder = topology.internalTopologyBuilder;
+        internalTopologyBuilder = builder;
         internalTopologyBuilder.setApplicationId(streamsConfig.getString(StreamsConfig.APPLICATION_ID_CONFIG));
 
         processorTopology = internalTopologyBuilder.build(null);

From 8725e3604b31d0bf822959fc6d870512411c7a05 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 26 Apr 2018 13:16:51 -0700
Subject: [PATCH 0300/1847] MINOR: Remove deprecated streams config (#4906)

Removed the following: "zookeeper.connect", "key.serde", "value.serde", "timestamp.extractor"

Reviewers: Bill Bejeck , John Roesler , Jason Gustafson 
---
 .../apache/kafka/streams/StreamsConfig.java   | 105 +-----------------
 .../kafka/streams/StreamsConfigTest.java      |  47 +-------
 2 files changed, 8 insertions(+), 144 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index e46d6d086ea24..23e69c5cbc733 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -279,14 +279,6 @@ public class StreamsConfig extends AbstractConfig {
     public static final String DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG = "default.timestamp.extractor";
     private static final String DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_DOC = "Default timestamp extractor class that implements the org.apache.kafka.streams.processor.TimestampExtractor interface.";
 
-    /**
-     * {@code key.serde}
-     * @deprecated Use {@link #DEFAULT_KEY_SERDE_CLASS_CONFIG} instead.
-     */
-    @Deprecated
-    public static final String KEY_SERDE_CLASS_CONFIG = "key.serde";
-    private static final String KEY_SERDE_CLASS_DOC = "Serializer / deserializer class for key that implements the org.apache.kafka.common.serialization.Serde interface. This config is deprecated, use " + DEFAULT_KEY_SERDE_CLASS_CONFIG + " instead";
-
     /** {@code metadata.max.age.ms} */
     public static final String METADATA_MAX_AGE_CONFIG = CommonClientConfigs.METADATA_MAX_AGE_CONFIG;
 
@@ -363,40 +355,16 @@ public class StreamsConfig extends AbstractConfig {
     public static final String STATE_DIR_CONFIG = "state.dir";
     private static final String STATE_DIR_DOC = "Directory location for state store.";
 
-    /**
-     * {@code timestamp.extractor}
-     * @deprecated Use {@link #DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG} instead.
-     */
-    @Deprecated
-    public static final String TIMESTAMP_EXTRACTOR_CLASS_CONFIG = "timestamp.extractor";
-    private static final String TIMESTAMP_EXTRACTOR_CLASS_DOC = "Timestamp extractor class that implements the org.apache.kafka.streams.processor.TimestampExtractor interface. This config is deprecated, use " + DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG + " instead";
-
     /** {@code upgrade.from} */
     public static final String UPGRADE_FROM_CONFIG = "upgrade.from";
     public static final String UPGRADE_FROM_DOC = "Allows upgrading from versions 0.10.0/0.10.1/0.10.2/0.11.0/1.0/1.1 to version 1.2 (or newer) in a backward compatible way. " +
         "When upgrading from 1.2 to a newer version it is not required to specify this config." +
         "Default is null. Accepted values are \"" + UPGRADE_FROM_0100 + "\", \"" + UPGRADE_FROM_0101 + "\", \"" + UPGRADE_FROM_0102 + "\", \"" + UPGRADE_FROM_0110 + "\", \"" + UPGRADE_FROM_10 + "\", \"" + UPGRADE_FROM_11 + "\" (for upgrading from the corresponding old version).";
 
-    /**
-     * {@code value.serde}
-     * @deprecated Use {@link #DEFAULT_VALUE_SERDE_CLASS_CONFIG} instead.
-     */
-    @Deprecated
-    public static final String VALUE_SERDE_CLASS_CONFIG = "value.serde";
-    private static final String VALUE_SERDE_CLASS_DOC = "Serializer / deserializer class for value that implements the org.apache.kafka.common.serialization.Serde interface. This config is deprecated, use " + DEFAULT_VALUE_SERDE_CLASS_CONFIG + " instead";
-
     /** {@code windowstore.changelog.additional.retention.ms} */
     public static final String WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG = "windowstore.changelog.additional.retention.ms";
     private static final String WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_DOC = "Added to a windows maintainMs to ensure data is not deleted from the log prematurely. Allows for clock drift. Default is 1 day";
 
-    /**
-     * {@code zookeeper.connect}
-     * @deprecated Kafka Streams does not use Zookeeper anymore and this parameter will be ignored.
-     */
-    @Deprecated
-    public static final String ZOOKEEPER_CONNECT_CONFIG = "zookeeper.connect";
-    private static final String ZOOKEEPER_CONNECT_DOC = "Zookeeper connect string for Kafka topics management. This config is deprecated and will be ignored as Streams API does not use Zookeeper anymore.";
-
     private static final String[] NON_CONFIGURABLE_CONSUMER_DEFAULT_CONFIGS = new String[] {ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG};
     private static final String[] NON_CONFIGURABLE_CONSUMER_EOS_CONFIGS = new String[] {ConsumerConfig.ISOLATION_LEVEL_CONFIG};
     private static final String[] NON_CONFIGURABLE_PRODUCER_EOS_CONFIGS = new String[] {ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG,
@@ -609,30 +577,7 @@ public class StreamsConfig extends AbstractConfig {
                     Type.LONG,
                     24 * 60 * 60 * 1000L,
                     Importance.LOW,
-                    WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_DOC)
-
-            // @deprecated
-
-            .define(KEY_SERDE_CLASS_CONFIG,
-                    Type.CLASS,
-                    null,
-                    Importance.LOW,
-                    KEY_SERDE_CLASS_DOC)
-            .define(TIMESTAMP_EXTRACTOR_CLASS_CONFIG,
-                    Type.CLASS,
-                    null,
-                    Importance.LOW,
-                    TIMESTAMP_EXTRACTOR_CLASS_DOC)
-            .define(VALUE_SERDE_CLASS_CONFIG,
-                    Type.CLASS,
-                    null,
-                    Importance.LOW,
-                    VALUE_SERDE_CLASS_DOC)
-            .define(ZOOKEEPER_CONNECT_CONFIG,
-                    Type.STRING,
-                    "",
-                    Importance.LOW,
-                    ZOOKEEPER_CONNECT_DOC);
+                    WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_DOC);
     }
 
     // this is the list of configs for underlying clients
@@ -773,8 +718,6 @@ private Map getCommonConsumerConfigs() {
 
         // bootstrap.servers should be from StreamsConfig
         consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, originals().get(BOOTSTRAP_SERVERS_CONFIG));
-        // remove deprecate ZK config
-        consumerProps.remove(ZOOKEEPER_CONNECT_CONFIG);
 
         return consumerProps;
     }
@@ -968,17 +911,6 @@ private Map getClientCustomProps() {
         return props;
     }
 
-    /**
-     * Return an {@link Serde#configure(Map, boolean) configured} instance of {@link #KEY_SERDE_CLASS_CONFIG key Serde
-     * class}. This method is deprecated. Use {@link #defaultKeySerde()} method instead.
-     *
-     * @return an configured instance of key Serde class
-     */
-    @Deprecated
-    public Serde keySerde() {
-        return defaultKeySerde();
-    }
-
     /**
      * Return an {@link Serde#configure(Map, boolean) configured} instance of {@link #DEFAULT_KEY_SERDE_CLASS_CONFIG key Serde
      * class}.
@@ -986,13 +918,9 @@ public Serde keySerde() {
      * @return an configured instance of key Serde class
      */
     public Serde defaultKeySerde() {
-        Object keySerdeConfigSetting = get(KEY_SERDE_CLASS_CONFIG);
+        Object keySerdeConfigSetting = get(DEFAULT_KEY_SERDE_CLASS_CONFIG);
         try {
-            Serde serde = getConfiguredInstance(KEY_SERDE_CLASS_CONFIG, Serde.class);
-            if (serde == null) {
-                keySerdeConfigSetting = get(DEFAULT_KEY_SERDE_CLASS_CONFIG);
-                serde = getConfiguredInstance(DEFAULT_KEY_SERDE_CLASS_CONFIG, Serde.class);
-            }
+            Serde serde = getConfiguredInstance(DEFAULT_KEY_SERDE_CLASS_CONFIG, Serde.class);
             serde.configure(originals(), true);
             return serde;
         } catch (final Exception e) {
@@ -1001,17 +929,6 @@ public Serde defaultKeySerde() {
         }
     }
 
-    /**
-     * Return an {@link Serde#configure(Map, boolean) configured} instance of {@link #VALUE_SERDE_CLASS_CONFIG value
-     * Serde class}. This method is deprecated. Use {@link #defaultValueSerde()} instead.
-     *
-     * @return an configured instance of value Serde class
-     */
-    @Deprecated
-    public Serde valueSerde() {
-        return defaultValueSerde();
-    }
-
     /**
      * Return an {@link Serde#configure(Map, boolean) configured} instance of {@link #DEFAULT_VALUE_SERDE_CLASS_CONFIG value
      * Serde class}.
@@ -1019,15 +936,10 @@ public Serde valueSerde() {
      * @return an configured instance of value Serde class
      */
     public Serde defaultValueSerde() {
-        Object valueSerdeConfigSetting = get(VALUE_SERDE_CLASS_CONFIG);
+        Object valueSerdeConfigSetting = get(DEFAULT_VALUE_SERDE_CLASS_CONFIG);
         try {
-            Serde serde = getConfiguredInstance(VALUE_SERDE_CLASS_CONFIG, Serde.class);
-            if (serde == null) {
-                valueSerdeConfigSetting = get(DEFAULT_VALUE_SERDE_CLASS_CONFIG);
-                serde = getConfiguredInstance(DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serde.class);
-            }
+            Serde serde = getConfiguredInstance(DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serde.class);
             serde.configure(originals(), false);
-
             return serde;
         } catch (final Exception e) {
             throw new StreamsException(
@@ -1035,13 +947,8 @@ public Serde defaultValueSerde() {
         }
     }
 
-
     public TimestampExtractor defaultTimestampExtractor() {
-        TimestampExtractor timestampExtractor = getConfiguredInstance(TIMESTAMP_EXTRACTOR_CLASS_CONFIG, TimestampExtractor.class);
-        if (timestampExtractor == null) {
-            return getConfiguredInstance(DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, TimestampExtractor.class);
-        }
-        return timestampExtractor;
+        return getConfiguredInstance(DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, TimestampExtractor.class);
     }
 
     public DeserializationExceptionHandler defaultDeserializationExceptionHandler() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index ef5e5a86be757..ac82c049b5234 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -454,20 +454,6 @@ public void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled() {
         assertThat(streamsConfig.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG), equalTo(commitIntervalMs));
     }
 
-    @SuppressWarnings("deprecation")
-    @Test
-    public void shouldBeBackwardsCompatibleWithDeprecatedConfigs() {
-        final Properties props = minimalStreamsConfig();
-        props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.Double().getClass());
-        props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Double().getClass());
-        props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MockTimestampExtractor.class);
-
-        final StreamsConfig config = new StreamsConfig(props);
-        assertTrue(config.defaultKeySerde() instanceof Serdes.DoubleSerde);
-        assertTrue(config.defaultValueSerde() instanceof Serdes.DoubleSerde);
-        assertTrue(config.defaultTimestampExtractor() instanceof MockTimestampExtractor);
-    }
-
     @Test
     public void shouldUseNewConfigsWhenPresent() {
         final Properties props = minimalStreamsConfig();
@@ -489,48 +475,19 @@ public void shouldUseCorrectDefaultsWhenNoneSpecified() {
         assertTrue(config.defaultTimestampExtractor() instanceof FailOnInvalidTimestamp);
     }
 
-    @SuppressWarnings("deprecation")
-    @Test
-    public void shouldSpecifyCorrectKeySerdeClassOnErrorUsingDeprecatedConfigs() {
-        final Properties props = minimalStreamsConfig();
-        props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
-        final StreamsConfig config = new StreamsConfig(props);
-        try {
-            config.keySerde();
-            fail("Test should throw a StreamsException");
-        } catch (StreamsException e) {
-            assertEquals("Failed to configure key serde class org.apache.kafka.streams.StreamsConfigTest$MisconfiguredSerde", e.getMessage());
-        }
-    }
-
-    @SuppressWarnings("deprecation")
     @Test
     public void shouldSpecifyCorrectKeySerdeClassOnError() {
         final Properties props = minimalStreamsConfig();
         props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
         final StreamsConfig config = new StreamsConfig(props);
         try {
-            config.keySerde();
+            config.defaultKeySerde();
             fail("Test should throw a StreamsException");
         } catch (StreamsException e) {
             assertEquals("Failed to configure key serde class org.apache.kafka.streams.StreamsConfigTest$MisconfiguredSerde", e.getMessage());
         }
     }
 
-    @SuppressWarnings("deprecation")
-    @Test
-    public void shouldSpecifyCorrectValueSerdeClassOnErrorUsingDeprecatedConfigs() {
-        final Properties props = minimalStreamsConfig();
-        props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
-        final StreamsConfig config = new StreamsConfig(props);
-        try {
-            config.valueSerde();
-            fail("Test should throw a StreamsException");
-        } catch (StreamsException e) {
-            assertEquals("Failed to configure value serde class org.apache.kafka.streams.StreamsConfigTest$MisconfiguredSerde", e.getMessage());
-        }
-    }
-
     @SuppressWarnings("deprecation")
     @Test
     public void shouldSpecifyCorrectValueSerdeClassOnError() {
@@ -538,7 +495,7 @@ public void shouldSpecifyCorrectValueSerdeClassOnError() {
         props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
         final StreamsConfig config = new StreamsConfig(props);
         try {
-            config.valueSerde();
+            config.defaultValueSerde();
             fail("Test should throw a StreamsException");
         } catch (StreamsException e) {
             assertEquals("Failed to configure value serde class org.apache.kafka.streams.StreamsConfigTest$MisconfiguredSerde", e.getMessage());

From ff1875fce0a82737069e195060b6a93881954a23 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Fri, 27 Apr 2018 02:31:04 +0530
Subject: [PATCH 0301/1847] KAFKA-6778; AdminClient.describeConfigs() should
 return error for non-existent topics (#4866)

Reviewers: Ismael Juma , Jason Gustafson 
---
 .../scala/kafka/server/AdminManager.scala     | 12 ++++++----
 .../api/AdminClientIntegrationTest.scala      | 22 +++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index b54defc1517d2..01457a197319a 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -304,10 +304,14 @@ class AdminManager(val config: KafkaConfig,
           case ResourceType.TOPIC =>
             val topic = resource.name
             Topic.validate(topic)
-            // Consider optimizing this by caching the configs or retrieving them from the `Log` when possible
-            val topicProps = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic)
-            val logConfig = LogConfig.fromProps(KafkaServer.copyKafkaConfigToLog(config), topicProps)
-            createResponseConfig(allConfigs(logConfig), createTopicConfigEntry(logConfig, topicProps, includeSynonyms))
+            if (metadataCache.contains(topic)) {
+              // Consider optimizing this by caching the configs or retrieving them from the `Log` when possible
+              val topicProps = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic)
+              val logConfig = LogConfig.fromProps(KafkaServer.copyKafkaConfigToLog(config), topicProps)
+              createResponseConfig(allConfigs(logConfig), createTopicConfigEntry(logConfig, topicProps, includeSynonyms))
+            } else {
+              new DescribeConfigsResponse.Config(new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, null), Collections.emptyList[DescribeConfigsResponse.ConfigEntry])
+            }
 
           case ResourceType.BROKER =>
             if (resource.name == null || resource.name.isEmpty)
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 33c14c69fc21a..b31c09d78d358 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -49,6 +49,7 @@ import scala.collection.JavaConverters._
 import java.lang.{Long => JLong}
 
 import kafka.zk.KafkaZkClient
+import org.scalatest.Assertions.intercept
 
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future}
@@ -821,6 +822,27 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
     client.close()
   }
 
+  @Test
+  def testDescribeConfigsForTopic(): Unit = {
+    createTopic(topic, numPartitions = 2, replicationFactor = serverCount)
+    client = AdminClient.create(createConfig)
+
+    val existingTopic = new ConfigResource(ConfigResource.Type.TOPIC, topic)
+    client.describeConfigs(Collections.singletonList(existingTopic)).values.get(existingTopic).get()
+
+    val nonExistentTopic = new ConfigResource(ConfigResource.Type.TOPIC, "unknown")
+    val describeResult1 = client.describeConfigs(Collections.singletonList(nonExistentTopic))
+
+    assertTrue(intercept[ExecutionException](describeResult1.values.get(nonExistentTopic).get).getCause.isInstanceOf[UnknownTopicOrPartitionException])
+
+    val invalidTopic = new ConfigResource(ConfigResource.Type.TOPIC, "(invalid topic)")
+    val describeResult2 = client.describeConfigs(Collections.singletonList(invalidTopic))
+
+    assertTrue(intercept[ExecutionException](describeResult2.values.get(invalidTopic).get).getCause.isInstanceOf[InvalidTopicException])
+
+    client.close()
+  }
+
   private def subscribeAndWaitForAssignment(topic: String, consumer: KafkaConsumer[Array[Byte], Array[Byte]]): Unit = {
     consumer.subscribe(Collections.singletonList(topic))
     TestUtils.waitUntilTrue(() => {

From e128f7f3bb6f7761225b2c0f3dfe1bf1d5d1aff8 Mon Sep 17 00:00:00 2001
From: Max Zheng 
Date: Thu, 26 Apr 2018 21:01:27 -0700
Subject: [PATCH 0302/1847] MINOR: Pin pip to 9.0.3 as 10 is not compatible
 with with system pip (#4937)

If not pinned, the following error will happen:
Traceback (most recent call last):
  File "/usr/bin/pip", line 9, in 
    from pip import main
ImportError: cannot import name main

Reviewers: Guozhang Wang 
---
 tests/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 25da8db59eaf2..f1239a919e5b7 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -32,7 +32,7 @@ LABEL ducker.creator=$ducker_creator
 
 # Update Linux and install necessary utilities.
 RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev && apt-get -y clean
-RUN pip install -U pip setuptools && pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.1
+RUN pip install -U pip==9.0.3 setuptools && pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.1
 
 # Set up ssh
 COPY ./ssh-config /root/.ssh/config

From b2e0812f6946f88cef2cdffb0be603b4b769033a Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 27 Apr 2018 13:40:24 -0700
Subject: [PATCH 0303/1847] HOTFIX: rename run_test to execute in streams
 simple benchmark (#4941)

---
 .../streams/streams_simple_benchmark_test.py         | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/kafkatest/benchmarks/streams/streams_simple_benchmark_test.py b/tests/kafkatest/benchmarks/streams/streams_simple_benchmark_test.py
index 6e6d676ead69e..b1cfb4e434c05 100644
--- a/tests/kafkatest/benchmarks/streams/streams_simple_benchmark_test.py
+++ b/tests/kafkatest/benchmarks/streams/streams_simple_benchmark_test.py
@@ -107,22 +107,22 @@ def test_simple_benchmark(self, test, scale):
 
         if test == ALL_TEST:
             for single_test in STREAMS_SIMPLE_TESTS + STREAMS_COUNT_TESTS + STREAMS_JOIN_TESTS:
-                self.run_test(single_test, scale)
+                self.execute(single_test, scale)
         elif test == STREAMS_SIMPLE_TEST:
             for single_test in STREAMS_SIMPLE_TESTS:
-                self.run_test(single_test, scale)
+                self.execute(single_test, scale)
         elif test == STREAMS_COUNT_TEST:
             for single_test in STREAMS_COUNT_TESTS:
-                self.run_test(single_test, scale)
+                self.execute(single_test, scale)
         elif test == STREAMS_JOIN_TEST:
             for single_test in STREAMS_JOIN_TESTS:
-                self.run_test(single_test, scale)
+                self.execute(single_test, scale)
         else:
-            self.run_test(test, scale)
+            self.execute(test, scale)
 
         return self.final
 
-    def run_test(self, test, scale):
+    def execute(self, test, scale):
 
         ################
         # RUN PHASE

From 3ce14a84a8c2009198e81a077cabe4b3031f6698 Mon Sep 17 00:00:00 2001
From: manjuapu 
Date: Fri, 27 Apr 2018 14:20:29 -0700
Subject: [PATCH 0304/1847] MINOR: Streams web doc table fix (#4942)

---
 docs/streams/developer-guide/dsl-api.html | 8 --------
 docs/streams/index.html                   | 3 +++
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/docs/streams/developer-guide/dsl-api.html b/docs/streams/developer-guide/dsl-api.html
index ad51884dc4c0b..a36bf8f0f0cef 100644
--- a/docs/streams/developer-guide/dsl-api.html
+++ b/docs/streams/developer-guide/dsl-api.html
@@ -1491,14 +1491,6 @@ 
    Overviewwindowed joins or
                         non-windowed joins.
    
                     
-                        -                            -                            -                            -                            -                            -                            -                        
diff --git a/docs/streams/index.html b/docs/streams/index.html
index 7ac6867ab247c..8992fc5e4f3f6 100644
--- a/docs/streams/index.html
+++ b/docs/streams/index.html
@@ -15,6 +15,9 @@
 
+
 
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java
index bd24e849ef5a5..234d3fc7e2ebb 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java
@@ -28,16 +28,14 @@
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Serialized;
 import org.apache.kafka.streams.kstream.TimeWindows;
-import org.apache.kafka.streams.kstream.ValueJoiner;
-import org.apache.kafka.streams.kstream.Windowed;
 
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -83,7 +81,7 @@ static public class RegionCount {
         public String region;
     }
 
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) {
         Properties props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
@@ -151,56 +149,56 @@ public static void main(String[] args) throws Exception {
                                                           Consumed.with(Serdes.String(), userProfileSerde));
 
         KStream regionCount = views
-            .leftJoin(users, new ValueJoiner() {
-                @Override
-                public PageViewByRegion apply(PageView view, UserProfile profile) {
-                    PageViewByRegion viewByRegion = new PageViewByRegion();
-                    viewByRegion.user = view.user;
-                    viewByRegion.page = view.page;
-
-                    if (profile != null) {
-                        viewByRegion.region = profile.region;
-                    } else {
-                        viewByRegion.region = "UNKNOWN";
-                    }
-                    return viewByRegion;
-                }
-            })
-            .map(new KeyValueMapper>() {
-                @Override
-                public KeyValue apply(String user, PageViewByRegion viewRegion) {
-                    return new KeyValue<>(viewRegion.region, viewRegion);
+            .leftJoin(users, (view, profile) -> {
+                PageViewByRegion viewByRegion = new PageViewByRegion();
+                viewByRegion.user = view.user;
+                viewByRegion.page = view.page;
+
+                if (profile != null) {
+                    viewByRegion.region = profile.region;
+                } else {
+                    viewByRegion.region = "UNKNOWN";
                 }
+                return viewByRegion;
             })
+            .map((user, viewRegion) -> new KeyValue<>(viewRegion.region, viewRegion))
             .groupByKey(Serialized.with(Serdes.String(), pageViewByRegionSerde))
             .windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(7)).advanceBy(TimeUnit.SECONDS.toMillis(1)))
             .count()
             .toStream()
-            .map(new KeyValueMapper, Long, KeyValue>() {
-                @Override
-                public KeyValue apply(Windowed key, Long value) {
-                    WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
-                    wViewByRegion.windowStart = key.window().start();
-                    wViewByRegion.region = key.key();
-
-                    RegionCount rCount = new RegionCount();
-                    rCount.region = key.key();
-                    rCount.count = value;
-
-                    return new KeyValue<>(wViewByRegion, rCount);
-                }
+            .map((key, value) -> {
+                WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
+                wViewByRegion.windowStart = key.window().start();
+                wViewByRegion.region = key.key();
+
+                RegionCount rCount = new RegionCount();
+                rCount.region = key.key();
+                rCount.count = value;
+
+                return new KeyValue<>(wViewByRegion, rCount);
             });
 
         // write to the result topic
         regionCount.to("streams-pageviewstats-typed-output", Produced.with(wPageViewByRegionSerde, regionCountSerde));
 
         KafkaStreams streams = new KafkaStreams(builder.build(), props);
-        streams.start();
-
-        // usually the stream application would be running forever,
-        // in this example we just let it run for some time and stop since the input data is finite.
-        Thread.sleep(5000L);
-
-        streams.close();
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        // attach shutdown handler to catch control-c
+        Runtime.getRuntime().addShutdownHook(new Thread("streams-pipe-shutdown-hook") {
+            @Override
+            public void run() {
+                streams.close();
+                latch.countDown();
+            }
+        });
+
+        try {
+            streams.start();
+            latch.await();
+        } catch (Throwable e) {
+            System.exit(1);
+        }
+        System.exit(0);
     }
 }
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewUntypedDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewUntypedDemo.java
index c38d68509a378..dddb542cbaff5 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewUntypedDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewUntypedDemo.java
@@ -33,13 +33,9 @@
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Serialized;
 import org.apache.kafka.streams.kstream.TimeWindows;
-import org.apache.kafka.streams.kstream.ValueJoiner;
-import org.apache.kafka.streams.kstream.ValueMapper;
-import org.apache.kafka.streams.kstream.Windowed;
 
 import java.util.Properties;
 
@@ -79,46 +75,30 @@ public static void main(String[] args) throws Exception {
 
         KTable users = builder.table("streams-userprofile-input", consumed);
 
-        KTable userRegions = users.mapValues(new ValueMapper() {
-            @Override
-            public String apply(JsonNode record) {
-                return record.get("region").textValue();
-            }
-        });
+        KTable userRegions = users.mapValues(record -> record.get("region").textValue());
 
         KStream regionCount = views
-            .leftJoin(userRegions, new ValueJoiner() {
-                @Override
-                public JsonNode apply(JsonNode view, String region) {
-                    ObjectNode jNode = JsonNodeFactory.instance.objectNode();
-
-                    return jNode.put("user", view.get("user").textValue())
-                            .put("page", view.get("page").textValue())
-                            .put("region", region == null ? "UNKNOWN" : region);
-                }
-            })
-            .map(new KeyValueMapper>() {
-                @Override
-                public KeyValue apply(String user, JsonNode viewRegion) {
-                    return new KeyValue<>(viewRegion.get("region").textValue(), viewRegion);
-                }
+            .leftJoin(userRegions, (view, region) -> {
+                ObjectNode jNode = JsonNodeFactory.instance.objectNode();
+                return (JsonNode) jNode.put("user", view.get("user").textValue())
+                        .put("page", view.get("page").textValue())
+                        .put("region", region == null ? "UNKNOWN" : region);
+
             })
+            .map((user, viewRegion) -> new KeyValue<>(viewRegion.get("region").textValue(), viewRegion))
             .groupByKey(Serialized.with(Serdes.String(), jsonSerde))
             .windowedBy(TimeWindows.of(7 * 24 * 60 * 60 * 1000L).advanceBy(1000))
             .count()
             .toStream()
-            .map(new KeyValueMapper, Long, KeyValue>() {
-                @Override
-                public KeyValue apply(Windowed key, Long value) {
-                    ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
-                    keyNode.put("window-start", key.window().start())
-                            .put("region", key.key());
-
-                    ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
-                    valueNode.put("count", value);
-
-                    return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
-                }
+            .map((key, value) -> {
+                ObjectNode keyNode = JsonNodeFactory.instance.objectNode();
+                keyNode.put("window-start", key.window().start())
+                        .put("region", key.key());
+
+                ObjectNode valueNode = JsonNodeFactory.instance.objectNode();
+                valueNode.put("count", value);
+
+                return new KeyValue<>((JsonNode) keyNode, (JsonNode) valueNode);
             });
 
         // write to the result topic
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pipe/PipeDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pipe/PipeDemo.java
index 538987747c9e2..d61e174a4f42f 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pipe/PipeDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pipe/PipeDemo.java
@@ -38,7 +38,7 @@
  */
 public class PipeDemo {
 
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) {
         Properties props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pipe");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/temperature/TemperatureDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/temperature/TemperatureDemo.java
index c5eb5f9ec7d89..4607d75d1387b 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/temperature/TemperatureDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/temperature/TemperatureDemo.java
@@ -23,10 +23,7 @@
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
-import org.apache.kafka.streams.kstream.Predicate;
 import org.apache.kafka.streams.kstream.Produced;
-import org.apache.kafka.streams.kstream.Reducer;
 import org.apache.kafka.streams.kstream.TimeWindows;
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.kstream.WindowedSerdes;
@@ -71,7 +68,7 @@ public class TemperatureDemo {
     // window size within which the filtering is applied
     private static final int TEMPERATURE_WINDOW_SIZE = 5;
 
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) {
 
         Properties props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-temperature");
@@ -89,30 +86,17 @@ public static void main(String[] args) throws Exception {
         KStream, String> max = source
             // temperature values are sent without a key (null), so in order
             // to group and reduce them, a key is needed ("temp" has been chosen)
-            .selectKey(new KeyValueMapper() {
-                @Override
-                public String apply(String key, String value) {
-                    return "temp";
-                }
-            })
+            .selectKey((key, value) -> "temp")
             .groupByKey()
             .windowedBy(TimeWindows.of(TimeUnit.SECONDS.toMillis(TEMPERATURE_WINDOW_SIZE)))
-            .reduce(new Reducer() {
-                @Override
-                public String apply(String value1, String value2) {
-                    if (Integer.parseInt(value1) > Integer.parseInt(value2))
-                        return value1;
-                    else
-                        return value2;
-                }
+            .reduce((value1, value2) -> {
+                if (Integer.parseInt(value1) > Integer.parseInt(value2))
+                    return value1;
+                else
+                    return value2;
             })
             .toStream()
-            .filter(new Predicate, String>() {
-                @Override
-                public boolean test(Windowed key, String value) {
-                    return Integer.parseInt(value) > TEMPERATURE_THRESHOLD;
-                }
-            });
+            .filter((key, value) -> Integer.parseInt(value) > TEMPERATURE_THRESHOLD);
 
         Serde> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class);
 
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
index 7535315d04f9c..4f0150e00582a 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java
@@ -23,9 +23,7 @@
 import org.apache.kafka.streams.StreamsConfig;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.kstream.KeyValueMapper;
 import org.apache.kafka.streams.kstream.Produced;
-import org.apache.kafka.streams.kstream.ValueMapper;
 
 import java.util.Arrays;
 import java.util.Locale;
@@ -46,7 +44,7 @@
  */
 public class WordCountDemo {
 
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) {
         Properties props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
@@ -64,18 +62,8 @@ public static void main(String[] args) throws Exception {
         KStream source = builder.stream("streams-plaintext-input");
 
         KTable counts = source
-            .flatMapValues(new ValueMapper>() {
-                @Override
-                public Iterable apply(String value) {
-                    return Arrays.asList(value.toLowerCase(Locale.getDefault()).split(" "));
-                }
-            })
-            .groupBy(new KeyValueMapper() {
-                @Override
-                public String apply(String key, String value) {
-                    return value;
-                }
-            })
+            .flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split(" ")))
+            .groupBy((key, value) -> value)
             .count();
 
         // need to override value serde to Long type
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java
index 523bb466e4473..86feaeb1ede66 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java
@@ -26,7 +26,6 @@
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.processor.ProcessorSupplier;
 import org.apache.kafka.streams.processor.PunctuationType;
-import org.apache.kafka.streams.processor.Punctuator;
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.Stores;
@@ -61,19 +60,16 @@ public Processor get() {
                 @SuppressWarnings("unchecked")
                 public void init(final ProcessorContext context) {
                     this.context = context;
-                    this.context.schedule(1000, PunctuationType.STREAM_TIME, new Punctuator() {
-                        @Override
-                        public void punctuate(long timestamp) {
-                            try (KeyValueIterator iter = kvStore.all()) {
-                                System.out.println("----------- " + timestamp + " ----------- ");
+                    this.context.schedule(1000, PunctuationType.STREAM_TIME, timestamp -> {
+                        try (KeyValueIterator iter = kvStore.all()) {
+                            System.out.println("----------- " + timestamp + " ----------- ");
 
-                                while (iter.hasNext()) {
-                                    KeyValue entry = iter.next();
+                            while (iter.hasNext()) {
+                                KeyValue entry = iter.next();
 
-                                    System.out.println("[" + entry.key + ", " + entry.value + "]");
+                                System.out.println("[" + entry.key + ", " + entry.value + "]");
 
-                                    context.forward(entry.key, entry.value.toString());
-                                }
+                                context.forward(entry.key, entry.value.toString());
                             }
                         }
                     });
@@ -103,7 +99,7 @@ public void close() {}
         }
     }
 
-    public static void main(String[] args) throws Exception {
+    public static void main(String[] args) {
         Properties props = new Properties();
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount-processor");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
@@ -123,7 +119,7 @@ public static void main(String[] args) throws Exception {
                 Stores.inMemoryKeyValueStore("Counts"),
                 Serdes.String(),
                 Serdes.Integer()),
-                              "Process");
+                "Process");
 
         builder.addSink("Sink", "streams-wordcount-processor-output", "Process");
 
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java
index ec40d2aad7548..bbf54e6a8cb79 100644
--- a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/LineSplit.java
@@ -44,24 +44,10 @@ public static void main(String[] args) throws Exception {
 
         final StreamsBuilder builder = new StreamsBuilder();
 
-        builder.stream("streams-plaintext-input")
-               .flatMapValues(new ValueMapper>() {
-                    @Override
-                    public Iterable apply(String value) {
-                        return Arrays.asList(value.split("\\W+"));
-                    }
-                })
-               .to("streams-linesplit-output");
-
-        /* ------- use the code below for Java 8 and uncomment the above ----
-
         builder.stream("streams-plaintext-input")
                .flatMapValues(value -> Arrays.asList(value.split("\\W+")))
                .to("streams-linesplit-output");
 
-           ----------------------------------------------------------------- */
-
-
         final Topology topology = builder.build();
         final KafkaStreams streams = new KafkaStreams(topology, props);
         final CountDownLatch latch = new CountDownLatch(1);
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/WordCount.java b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/WordCount.java
index 020eb03ca2b41..bdbefed3be314 100644
--- a/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/WordCount.java
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/src/main/java/WordCount.java
@@ -50,26 +50,6 @@ public static void main(String[] args) throws Exception {
 
         final StreamsBuilder builder = new StreamsBuilder();
 
-        builder.stream("streams-plaintext-input")
-               .flatMapValues(new ValueMapper>() {
-                    @Override
-                    public Iterable apply(String value) {
-                        return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
-                    }
-                })
-               .groupBy(new KeyValueMapper() {
-                   @Override
-                   public String apply(String key, String value) {
-                       return value;
-                   }
-                })
-               .count(Materialized.>as("counts-store"))
-               .toStream()
-               .to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()));
-
-
-        /* ------- use the code below for Java 8 and comment the above ----
-
         builder.stream("streams-plaintext-input")
                .flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+")))
                .groupBy((key, value) -> value)
@@ -77,8 +57,6 @@ public String apply(String key, String value) {
                .toStream()
                .to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()));
 
-           ----------------------------------------------------------------- */
-
         final Topology topology = builder.build();
         final KafkaStreams streams = new KafkaStreams(topology, props);
         final CountDownLatch latch = new CountDownLatch(1);

From e33ccb628efd9a039d2ad9f9ca74f001bc682fcd Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Thu, 21 Jun 2018 16:47:44 -0700
Subject: [PATCH 0548/1847] KAFKA-7082: Concurrent create topics may throw
 NodeExistsException (#5259)

This is an unexpected exception so `UnknownServerException`
is thrown back to the client.

This is a minimal change to make the behaviour match `ZkUtils`.
This is better, but one could argue that it's not perfect. A more
sophisticated approach can be tackled separately.

Added a concurrent test that fails without this change.

Reviewers: Jun Rao 
---
 .../main/scala/kafka/zk/AdminZkClient.scala   |  1 -
 .../main/scala/kafka/zk/KafkaZkClient.scala   | 15 ++++++++---
 .../unit/kafka/zk/AdminZkClientTest.scala     | 26 ++++++++++++++++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala
index 8a6b3ee212d3f..060c0b4d4aec8 100644
--- a/core/src/main/scala/kafka/zk/AdminZkClient.scala
+++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala
@@ -93,7 +93,6 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging {
                                                      update: Boolean = false) {
     validateCreateOrUpdateTopic(topic, partitionReplicaAssignment, config, update)
 
-    // Configs only matter if a topic is being created. Changing configs via AlterTopic is not supported
     if (!update) {
       // write out the config if there is any, this isn't transactional with the partition assignments
       zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic, config)
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index bb342945ea834..ec4932ab47bfb 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -246,6 +246,12 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Sets or creates the entity znode path with the given configs depending
    * on whether it already exists or not.
+   *
+   * If this is method is called concurrently, the last writer wins. In cases where we update configs and then
+   * partition assignment (i.e. create topic), it's possible for one thread to set this and the other to set the
+   * partition assignment. As such, the recommendation is to never call create topic for the same topic with different
+   * configs/partition assignment concurrently.
+   *
    * @param rootEntityType entity type
    * @param sanitizedEntityName entity name
    * @throws KeeperException if there is an error while setting or creating the znode
@@ -257,16 +263,19 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
       retryRequestUntilConnected(setDataRequest)
     }
 
-    def create(configData: Array[Byte]) = {
+    def createOrSet(configData: Array[Byte]): Unit = {
       val path = ConfigEntityZNode.path(rootEntityType, sanitizedEntityName)
-      createRecursive(path, ConfigEntityZNode.encode(config))
+      try createRecursive(path, ConfigEntityZNode.encode(config))
+      catch {
+        case _: NodeExistsException => set(configData).maybeThrow
+      }
     }
 
     val configData = ConfigEntityZNode.encode(config)
 
     val setDataResponse = set(configData)
     setDataResponse.resultCode match {
-      case Code.NONODE => create(configData)
+      case Code.NONODE => createOrSet(configData)
       case _ => setDataResponse.maybeThrow
     }
   }
diff --git a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
index fe5fbff55d05d..39745e5e608dd 100644
--- a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala
@@ -28,8 +28,10 @@ import kafka.utils.TestUtils._
 import kafka.utils.{Logging, TestUtils}
 import kafka.zk.{AdminZkClient, KafkaZkClient, ZooKeeperTestHarness}
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.config.TopicConfig
 import org.apache.kafka.common.errors.{InvalidReplicaAssignmentException, InvalidTopicException, TopicExistsException}
 import org.apache.kafka.common.metrics.Quota
+import org.apache.kafka.test.{TestUtils => JTestUtils}
 import org.easymock.EasyMock
 import org.junit.Assert._
 import org.junit.{After, Test}
@@ -132,7 +134,7 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
   }
 
   @Test
-  def testConcurrentTopicCreation() {
+  def testMockedConcurrentTopicCreation() {
     val topic = "test.topic"
 
     // simulate the ZK interactions that can happen when a topic is concurrently created by multiple processes
@@ -147,6 +149,28 @@ class AdminZkClientTest extends ZooKeeperTestHarness with Logging with RackAware
     }
   }
 
+  @Test
+  def testConcurrentTopicCreation() {
+    val topic = "test-concurrent-topic-creation"
+    TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4))
+    val props = new Properties
+    props.setProperty(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2")
+    def createTopic(): Unit = {
+      try adminZkClient.createTopic(topic, 3, 1, props)
+      catch { case _: TopicExistsException => () }
+      val (_, partitionAssignment) = zkClient.getPartitionAssignmentForTopics(Set(topic)).head
+      assertEquals(3, partitionAssignment.size)
+      partitionAssignment.foreach { case (partition, replicas) =>
+        assertEquals(s"Unexpected replication factor for $partition", 1, replicas.size)
+      }
+      val savedProps = zkClient.getEntityConfigs(ConfigType.Topic, topic)
+      assertEquals(props, savedProps)
+    }
+
+    TestUtils.assertConcurrent("Concurrent topic creation failed", Seq(createTopic, createTopic),
+      JTestUtils.DEFAULT_MAX_WAIT_MS.toInt)
+  }
+
   /**
    * This test creates a topic with a few config overrides and checks that the configs are applied to the new topic
    * then changes the config and checks that the new values take effect.

From 418a91b5d4e3a0579b91d286f61c2b63c5b4a9b6 Mon Sep 17 00:00:00 2001
From: Vahid Hashemian 
Date: Thu, 21 Jun 2018 17:19:24 -0700
Subject: [PATCH 0549/1847] KAFKA-4682; Revise expiration semantics of consumer
 group offsets (KIP-211 - Part 1) (#4896)

This patch contains the improved offset expiration semantics proposed in KIP-211. Committed offsets will not be expired as long as a group is active. Once all members have left the group, then offsets will be expired after the timeout configured by `offsets.retention.minutes`. Note that the optimization for early expiration of unsubscribed topics will be implemented in a separate patch.
---
 .../internals/ConsumerCoordinator.java        |   3 +-
 .../common/requests/OffsetCommitRequest.java  |  20 +-
 .../common/requests/OffsetCommitResponse.java |   4 +-
 .../common/requests/RequestResponseTest.java  |  14 +-
 .../src/main/scala/kafka/api/ApiVersion.scala |  11 +-
 .../kafka/common/OffsetMetadataAndError.scala |   6 +-
 .../coordinator/group/GroupCoordinator.scala  |   6 +-
 .../coordinator/group/GroupMetadata.scala     |  62 ++-
 .../group/GroupMetadataManager.scala          | 151 +++++---
 .../main/scala/kafka/server/KafkaApis.scala   |  30 +-
 .../scala/kafka/tools/ConsoleConsumer.scala   |   4 +
 .../scala/kafka/tools/DumpLogSegments.scala   |   4 +-
 .../kafka/api/AuthorizerIntegrationTest.scala |   2 +-
 .../scala/unit/kafka/api/ApiVersionTest.scala |   3 +
 .../group/GroupMetadataManagerTest.scala      | 358 +++++++++++++++---
 .../coordinator/group/GroupMetadataTest.scala |   5 +-
 .../unit/kafka/server/RequestQuotaTest.scala  |   2 +-
 docs/upgrade.html                             |  19 +-
 18 files changed, 545 insertions(+), 159 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 9c19af1703779..b484e110aee30 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -798,8 +798,7 @@ private RequestFuture sendOffsetCommitRequest(final Map
         private final Map offsetData;
         private String memberId = DEFAULT_MEMBER_ID;
         private int generationId = DEFAULT_GENERATION_ID;
-        private long retentionTime = DEFAULT_RETENTION_TIME;
 
         public Builder(String groupId, Map offsetData) {
             super(ApiKeys.OFFSET_COMMIT);
@@ -184,11 +189,6 @@ public Builder setGenerationId(int generationId) {
             return this;
         }
 
-        public Builder setRetentionTime(long retentionTime) {
-            this.retentionTime = retentionTime;
-            return this;
-        }
-
         @Override
         public OffsetCommitRequest build(short version) {
             switch (version) {
@@ -199,8 +199,8 @@ public OffsetCommitRequest build(short version) {
                 case 2:
                 case 3:
                 case 4:
-                    long retentionTime = version == 1 ? DEFAULT_RETENTION_TIME : this.retentionTime;
-                    return new OffsetCommitRequest(groupId, generationId, memberId, retentionTime, offsetData, version);
+                case 5:
+                    return new OffsetCommitRequest(groupId, generationId, memberId, DEFAULT_RETENTION_TIME, offsetData, version);
                 default:
                     throw new UnsupportedVersionException("Unsupported version " + version);
             }
@@ -213,7 +213,6 @@ public String toString() {
                 append(", groupId=").append(groupId).
                 append(", memberId=").append(memberId).
                 append(", generationId=").append(generationId).
-                append(", retentionTime=").append(retentionTime).
                 append(", offsetData=").append(offsetData).
                 append(")");
             return bld.toString();
@@ -316,6 +315,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
                 return new OffsetCommitResponse(responseData);
             case 3:
             case 4:
+            case 5:
                 return new OffsetCommitResponse(throttleTimeMs, responseData);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index 0b0b228381828..c79bc57885b0b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -85,9 +85,11 @@ public class OffsetCommitResponse extends AbstractResponse {
      */
     private static final Schema OFFSET_COMMIT_RESPONSE_V4 = OFFSET_COMMIT_RESPONSE_V3;
 
+    private static final Schema OFFSET_COMMIT_RESPONSE_V5 = OFFSET_COMMIT_RESPONSE_V4;
+
     public static Schema[] schemaVersions() {
         return new Schema[] {OFFSET_COMMIT_RESPONSE_V0, OFFSET_COMMIT_RESPONSE_V1, OFFSET_COMMIT_RESPONSE_V2,
-            OFFSET_COMMIT_RESPONSE_V3, OFFSET_COMMIT_RESPONSE_V4};
+            OFFSET_COMMIT_RESPONSE_V3, OFFSET_COMMIT_RESPONSE_V4, OFFSET_COMMIT_RESPONSE_V5};
     }
 
     private final Map responseData;
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 6e705d2221093..da613982816c7 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -141,9 +141,6 @@ public void testSerialization() throws Exception {
         checkErrorResponse(createMetadataRequest(3, singletonList("topic1")), new UnknownServerException());
         checkResponse(createMetadataResponse(), 4);
         checkErrorResponse(createMetadataRequest(4, singletonList("topic1")), new UnknownServerException());
-        checkRequest(createOffsetCommitRequest(2));
-        checkErrorResponse(createOffsetCommitRequest(2), new UnknownServerException());
-        checkResponse(createOffsetCommitResponse(), 0);
         checkRequest(OffsetFetchRequest.forAllPartitions("group1"));
         checkErrorResponse(OffsetFetchRequest.forAllPartitions("group1"), new NotCoordinatorException("Not Coordinator"));
         checkRequest(createOffsetFetchRequest(0));
@@ -210,6 +207,16 @@ public void testSerialization() throws Exception {
         checkErrorResponse(createOffsetCommitRequest(0), new UnknownServerException());
         checkRequest(createOffsetCommitRequest(1));
         checkErrorResponse(createOffsetCommitRequest(1), new UnknownServerException());
+        checkRequest(createOffsetCommitRequest(2));
+        checkErrorResponse(createOffsetCommitRequest(2), new UnknownServerException());
+        checkRequest(createOffsetCommitRequest(3));
+        checkErrorResponse(createOffsetCommitRequest(3), new UnknownServerException());
+        checkRequest(createOffsetCommitRequest(4));
+        checkErrorResponse(createOffsetCommitRequest(4), new UnknownServerException());
+        checkResponse(createOffsetCommitResponse(), 4);
+        checkRequest(createOffsetCommitRequest(5));
+        checkErrorResponse(createOffsetCommitRequest(5), new UnknownServerException());
+        checkResponse(createOffsetCommitResponse(), 5);
         checkRequest(createJoinGroupRequest(0));
         checkRequest(createUpdateMetadataRequest(0, null));
         checkErrorResponse(createUpdateMetadataRequest(0, null), new UnknownServerException());
@@ -817,7 +824,6 @@ private OffsetCommitRequest createOffsetCommitRequest(int version) {
         return new OffsetCommitRequest.Builder("group1", commitData)
                 .setGenerationId(100)
                 .setMemberId("consumer1")
-                .setRetentionTime(1000000)
                 .build((short) version);
     }
 
diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala
index 9ed6432cbfd1d..485f2bdce31d0 100644
--- a/core/src/main/scala/kafka/api/ApiVersion.scala
+++ b/core/src/main/scala/kafka/api/ApiVersion.scala
@@ -73,7 +73,9 @@ object ApiVersion {
     // Introduced OffsetsForLeaderEpochRequest V1 via KIP-279
     KAFKA_2_0_IV0,
     // Introduced ApiVersionsRequest V2 via KIP-219
-    KAFKA_2_0_IV1
+    KAFKA_2_0_IV1,
+    // Introduced new schemas for group offset (v2) and group metadata (v2) (KIP-211)
+    KAFKA_2_1_IV0
   )
 
   // Map keys are the union of the short and full versions
@@ -248,4 +250,11 @@ case object KAFKA_2_0_IV1 extends DefaultApiVersion {
   val subVersion = "IV1"
   val recordVersion = RecordVersion.V2
   val id: Int = 16
+}
+
+case object KAFKA_2_1_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.1"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 18
 }
\ No newline at end of file
diff --git a/core/src/main/scala/kafka/common/OffsetMetadataAndError.scala b/core/src/main/scala/kafka/common/OffsetMetadataAndError.scala
index 2cf9bb40eec09..afe542c29c3e6 100644
--- a/core/src/main/scala/kafka/common/OffsetMetadataAndError.scala
+++ b/core/src/main/scala/kafka/common/OffsetMetadataAndError.scala
@@ -34,17 +34,17 @@ object OffsetMetadata {
 
 case class OffsetAndMetadata(offsetMetadata: OffsetMetadata,
                              commitTimestamp: Long = org.apache.kafka.common.requests.OffsetCommitRequest.DEFAULT_TIMESTAMP,
-                             expireTimestamp: Long = org.apache.kafka.common.requests.OffsetCommitRequest.DEFAULT_TIMESTAMP) {
+                             expireTimestamp: Option[Long] = None) {
 
   def offset = offsetMetadata.offset
 
   def metadata = offsetMetadata.metadata
 
-  override def toString = "[%s,CommitTime %d,ExpirationTime %d]".format(offsetMetadata, commitTimestamp, expireTimestamp)
+  override def toString = s"[$offsetMetadata,CommitTime $commitTimestamp,ExpirationTime ${expireTimestamp.getOrElse("_")}]"
 }
 
 object OffsetAndMetadata {
-  def apply(offset: Long, metadata: String, commitTimestamp: Long, expireTimestamp: Long) = new OffsetAndMetadata(OffsetMetadata(offset, metadata), commitTimestamp, expireTimestamp)
+  def apply(offset: Long, metadata: String, commitTimestamp: Long, expireTimestamp: Long) = new OffsetAndMetadata(OffsetMetadata(offset, metadata), commitTimestamp, Some(expireTimestamp))
 
   def apply(offset: Long, metadata: String, timestamp: Long) = new OffsetAndMetadata(OffsetMetadata(offset, metadata), timestamp)
 
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
index 9748e174c7847..2cedacd7200ca 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala
@@ -125,7 +125,7 @@ class GroupCoordinator(val brokerId: Int,
           if (memberId != JoinGroupRequest.UNKNOWN_MEMBER_ID) {
             responseCallback(joinError(memberId, Errors.UNKNOWN_MEMBER_ID))
           } else {
-            val group = groupManager.addGroup(new GroupMetadata(groupId, initialState = Empty))
+            val group = groupManager.addGroup(new GroupMetadata(groupId, Empty, time))
             doJoinGroup(group, memberId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, protocols, responseCallback)
           }
 
@@ -451,7 +451,7 @@ class GroupCoordinator(val brokerId: Int,
       case Some(error) => responseCallback(offsetMetadata.mapValues(_ => error))
       case None =>
         val group = groupManager.getGroup(groupId).getOrElse {
-          groupManager.addGroup(new GroupMetadata(groupId, initialState = Empty))
+          groupManager.addGroup(new GroupMetadata(groupId, Empty, time))
         }
         doCommitOffsets(group, NoMemberId, NoGeneration, producerId, producerEpoch, offsetMetadata, responseCallback)
     }
@@ -469,7 +469,7 @@ class GroupCoordinator(val brokerId: Int,
           case None =>
             if (generationId < 0) {
               // the group is not relying on Kafka for group management, so allow the commit
-              val group = groupManager.addGroup(new GroupMetadata(groupId, initialState = Empty))
+              val group = groupManager.addGroup(new GroupMetadata(groupId, Empty, time))
               doCommitOffsets(group, memberId, generationId, NO_PRODUCER_ID, NO_PRODUCER_EPOCH,
                 offsetMetadata, responseCallback)
             } else {
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index 2b9c91f61b73c..d729449af4e47 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.locks.ReentrantLock
 import kafka.common.OffsetAndMetadata
 import kafka.utils.{CoreUtils, Logging, nonthreadsafe}
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.utils.Time
 
 import scala.collection.{Seq, immutable, mutable}
 
@@ -118,12 +119,15 @@ private object GroupMetadata {
                 protocolType: String,
                 protocol: String,
                 leaderId: String,
-                members: Iterable[MemberMetadata]): GroupMetadata = {
-    val group = new GroupMetadata(groupId, initialState)
+                currentStateTimestamp: Option[Long],
+                members: Iterable[MemberMetadata],
+                time: Time): GroupMetadata = {
+    val group = new GroupMetadata(groupId, initialState, time)
     group.generationId = generationId
     group.protocolType = if (protocolType == null || protocolType.isEmpty) None else Some(protocolType)
     group.protocol = Option(protocol)
     group.leaderId = Option(leaderId)
+    group.currentStateTimestamp = currentStateTimestamp
     members.foreach(group.add)
     group
   }
@@ -167,10 +171,11 @@ case class CommitRecordMetadataAndOffset(appendedBatchOffset: Option[Long], offs
  *  3. leader id
  */
 @nonthreadsafe
-private[group] class GroupMetadata(val groupId: String, initialState: GroupState) extends Logging {
+private[group] class GroupMetadata(val groupId: String, initialState: GroupState, time: Time) extends Logging {
   private[group] val lock = new ReentrantLock
 
   private var state: GroupState = initialState
+  var currentStateTimestamp: Option[Long] = Some(time.milliseconds())
   var protocolType: Option[String] = None
   var generationId = 0
   private var leaderId: Option[String] = None
@@ -195,6 +200,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   def isLeader(memberId: String): Boolean = leaderId.contains(memberId)
   def leaderOrNull: String = leaderId.orNull
   def protocolOrNull: String = protocol.orNull
+  def currentStateTimestampOrDefault: Long = currentStateTimestamp.getOrElse(-1)
 
   def add(member: MemberMetadata) {
     if (members.isEmpty)
@@ -240,6 +246,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
   def transitionTo(groupState: GroupState) {
     assertValidTransition(groupState)
     state = groupState
+    currentStateTimestamp = Some(time.milliseconds())
   }
 
   def selectProtocol: String = {
@@ -434,18 +441,51 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
     }.toMap
   }
 
-  def removeExpiredOffsets(startMs: Long) : Map[TopicPartition, OffsetAndMetadata] = {
-    val expiredOffsets = offsets
-      .filter {
+  def removeExpiredOffsets(currentTimestamp: Long, offsetRetentionMs: Long) : Map[TopicPartition, OffsetAndMetadata] = {
+
+    def getExpiredOffsets(baseTimestamp: CommitRecordMetadataAndOffset => Long): Map[TopicPartition, OffsetAndMetadata] = {
+      offsets.filter {
         case (topicPartition, commitRecordMetadataAndOffset) =>
-          commitRecordMetadataAndOffset.offsetAndMetadata.expireTimestamp < startMs && !pendingOffsetCommits.contains(topicPartition)
-      }
-      .map {
+          !pendingOffsetCommits.contains(topicPartition) && {
+            commitRecordMetadataAndOffset.offsetAndMetadata.expireTimestamp match {
+              case None =>
+                // current version with no per partition retention
+                currentTimestamp - baseTimestamp(commitRecordMetadataAndOffset) >= offsetRetentionMs
+              case Some(expireTimestamp) =>
+                // older versions with explicit expire_timestamp field => old expiration semantics is used
+                currentTimestamp >= expireTimestamp
+            }
+          }
+      }.map {
         case (topicPartition, commitRecordOffsetAndMetadata) =>
           (topicPartition, commitRecordOffsetAndMetadata.offsetAndMetadata)
-      }
+      }.toMap
+    }
+
+    val expiredOffsets: Map[TopicPartition, OffsetAndMetadata] = protocolType match {
+      case Some(_) if is(Empty) =>
+        // no consumer exists in the group =>
+        // - if current state timestamp exists and retention period has passed since group became Empty,
+        //   expire all offsets with no pending offset commit;
+        // - if there is no current state timestamp (old group metadata schema) and retention period has passed
+        //   since the last commit timestamp, expire the offset
+        getExpiredOffsets(commitRecordMetadataAndOffset =>
+          currentStateTimestamp.getOrElse(commitRecordMetadataAndOffset.offsetAndMetadata.commitTimestamp))
+
+      case None =>
+        // protocolType is None => standalone (simple) consumer, that uses Kafka for offset storage only
+        // expire offsets with no pending offset commit that retention period has passed since their last commit
+        getExpiredOffsets(_.offsetAndMetadata.commitTimestamp)
+
+      case _ =>
+        Map()
+    }
+
+    if (expiredOffsets.nonEmpty)
+      debug(s"Expired offsets from group '$groupId': ${expiredOffsets.keySet}")
+
     offsets --= expiredOffsets.keySet
-    expiredOffsets.toMap
+    expiredOffsets
   }
 
   def allOffsets = offsets.map { case (topicPartition, commitRecordMetadataAndOffset) =>
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 02ba13a72b6f8..6bd0a5a0d52d6 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.locks.ReentrantLock
 
 import com.yammer.metrics.core.Gauge
-import kafka.api.{ApiVersion, KAFKA_0_10_1_IV0}
+import kafka.api.{ApiVersion, KAFKA_0_10_1_IV0, KAFKA_2_1_IV0}
 import kafka.common.{MessageFormatter, OffsetAndMetadata}
 import kafka.metrics.KafkaMetricsGroup
 import kafka.server.ReplicaManager
@@ -197,18 +197,11 @@ class GroupMetadataManager(brokerId: Int,
                  responseCallback: Errors => Unit): Unit = {
     getMagic(partitionFor(group.groupId)) match {
       case Some(magicValue) =>
-        val groupMetadataValueVersion = {
-          if (interBrokerProtocolVersion < KAFKA_0_10_1_IV0)
-            0.toShort
-          else
-            GroupMetadataManager.CURRENT_GROUP_VALUE_SCHEMA_VERSION
-        }
-
         // We always use CREATE_TIME, like the producer. The conversion to LOG_APPEND_TIME (if necessary) happens automatically.
         val timestampType = TimestampType.CREATE_TIME
         val timestamp = time.milliseconds()
         val key = GroupMetadataManager.groupMetadataKey(group.groupId)
-        val value = GroupMetadataManager.groupMetadataValue(group, groupAssignment, version = groupMetadataValueVersion)
+        val value = GroupMetadataManager.groupMetadataValue(group, groupAssignment, interBrokerProtocolVersion)
 
         val records = {
           val buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue, compressionType,
@@ -330,7 +323,7 @@ class GroupMetadataManager(brokerId: Int,
 
           val records = filteredOffsetMetadata.map { case (topicPartition, offsetAndMetadata) =>
             val key = GroupMetadataManager.offsetCommitKey(group.groupId, topicPartition)
-            val value = GroupMetadataManager.offsetCommitValue(offsetAndMetadata)
+            val value = GroupMetadataManager.offsetCommitValue(offsetAndMetadata, interBrokerProtocolVersion)
             new SimpleRecord(timestamp, key, value)
           }
           val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId))
@@ -580,7 +573,7 @@ class GroupMetadataManager(brokerId: Int,
                   case groupMetadataKey: GroupMetadataKey =>
                     // load group metadata
                     val groupId = groupMetadataKey.key
-                    val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value)
+                    val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value, time)
                     if (groupMetadata != null) {
                       removedGroups.remove(groupId)
                       loadedGroups.put(groupId, groupMetadata)
@@ -630,7 +623,7 @@ class GroupMetadataManager(brokerId: Int,
         // load groups which store offsets in kafka, but which have no active members and thus no group
         // metadata stored in the log
         (emptyGroupOffsets.keySet ++ pendingEmptyGroupOffsets.keySet).foreach { groupId =>
-          val group = new GroupMetadata(groupId, initialState = Empty)
+          val group = new GroupMetadata(groupId, Empty, time)
           val offsets = emptyGroupOffsets.getOrElse(groupId, Map.empty[TopicPartition, CommitRecordMetadataAndOffset])
           val pendingOffsets = pendingEmptyGroupOffsets.getOrElse(groupId, Map.empty[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]])
           debug(s"Loaded group metadata $group with offsets $offsets and pending offsets $pendingOffsets")
@@ -653,18 +646,8 @@ class GroupMetadataManager(brokerId: Int,
                         pendingTransactionalOffsets: Map[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]): Unit = {
     // offsets are initialized prior to loading the group into the cache to ensure that clients see a consistent
     // view of the group's offsets
-    val loadedOffsets = offsets.mapValues { case CommitRecordMetadataAndOffset(commitRecordOffset, offsetAndMetadata) =>
-      // special handling for version 0:
-      // set the expiration time stamp as commit time stamp + server default retention time
-      val updatedOffsetAndMetadata =
-        if (offsetAndMetadata.expireTimestamp == org.apache.kafka.common.requests.OffsetCommitRequest.DEFAULT_TIMESTAMP)
-        offsetAndMetadata.copy(expireTimestamp = offsetAndMetadata.commitTimestamp + config.offsetsRetentionMs)
-      else
-        offsetAndMetadata
-      CommitRecordMetadataAndOffset(commitRecordOffset, updatedOffsetAndMetadata)
-    }
-    trace(s"Initialized offsets $loadedOffsets for group ${group.groupId}")
-    group.initializeOffsets(loadedOffsets, pendingTransactionalOffsets.toMap)
+    trace(s"Initialized offsets $offsets for group ${group.groupId}")
+    group.initializeOffsets(offsets, pendingTransactionalOffsets.toMap)
 
     val currentGroup = addGroup(group)
     if (group != currentGroup)
@@ -711,11 +694,11 @@ class GroupMetadataManager(brokerId: Int,
 
   // visible for testing
   private[group] def cleanupGroupMetadata(): Unit = {
-    val startMs = time.milliseconds()
-    val offsetsRemoved = cleanupGroupMetadata(groupMetadataCache.values, group => {
-      group.removeExpiredOffsets(time.milliseconds())
+    val currentTimestamp = time.milliseconds()
+    val numOffsetsRemoved = cleanupGroupMetadata(groupMetadataCache.values, group => {
+      group.removeExpiredOffsets(currentTimestamp, config.offsetsRetentionMs)
     })
-    info(s"Removed $offsetsRemoved expired offsets in ${time.milliseconds() - startMs} milliseconds.")
+    info(s"Removed $numOffsetsRemoved expired offsets in ${time.milliseconds() - currentTimestamp} milliseconds.")
   }
 
   /**
@@ -917,7 +900,7 @@ class GroupMetadataManager(brokerId: Int,
  *    -> value version 1:       [offset, metadata, commit_timestamp, expire_timestamp]
  *
  * key version 2:       group metadata
- *     -> value version 0:       [protocol_type, generation, protocol, leader, members]
+ *    -> value version 0:       [protocol_type, generation, protocol, leader, members]
  */
 object GroupMetadataManager {
 
@@ -947,6 +930,13 @@ object GroupMetadataManager {
   private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get("commit_timestamp")
   private val OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get("expire_timestamp")
 
+  private val OFFSET_COMMIT_VALUE_SCHEMA_V2 = new Schema(new Field("offset", INT64),
+    new Field("metadata", STRING, "Associated metadata.", ""),
+    new Field("commit_timestamp", INT64))
+  private val OFFSET_VALUE_OFFSET_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get("offset")
+  private val OFFSET_VALUE_METADATA_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get("metadata")
+  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get("commit_timestamp")
+
   private val GROUP_METADATA_KEY_SCHEMA = new Schema(new Field("group", STRING))
   private val GROUP_KEY_GROUP_FIELD = GROUP_METADATA_KEY_SCHEMA.get("group")
 
@@ -975,10 +965,13 @@ object GroupMetadataManager {
     new Field(SUBSCRIPTION_KEY, BYTES),
     new Field(ASSIGNMENT_KEY, BYTES))
 
+  private val MEMBER_METADATA_V2 = MEMBER_METADATA_V1
+
   private val PROTOCOL_TYPE_KEY = "protocol_type"
   private val GENERATION_KEY = "generation"
   private val PROTOCOL_KEY = "protocol"
   private val LEADER_KEY = "leader"
+  private val CURRENT_STATE_TIMESTAMP_KEY = "current_state_timestamp"
   private val MEMBERS_KEY = "members"
 
   private val GROUP_METADATA_VALUE_SCHEMA_V0 = new Schema(
@@ -995,6 +988,14 @@ object GroupMetadataManager {
     new Field(LEADER_KEY, NULLABLE_STRING),
     new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V1)))
 
+  private val GROUP_METADATA_VALUE_SCHEMA_V2 = new Schema(
+    new Field(PROTOCOL_TYPE_KEY, STRING),
+    new Field(GENERATION_KEY, INT32),
+    new Field(PROTOCOL_KEY, NULLABLE_STRING),
+    new Field(LEADER_KEY, NULLABLE_STRING),
+    new Field(CURRENT_STATE_TIMESTAMP_KEY, INT64),
+    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V2)))
+
 
   // map of versions to key schemas as data types
   private val MESSAGE_TYPE_SCHEMAS = Map(
@@ -1005,19 +1006,20 @@ object GroupMetadataManager {
   // map of version of offset value schemas
   private val OFFSET_VALUE_SCHEMAS = Map(
     0 -> OFFSET_COMMIT_VALUE_SCHEMA_V0,
-    1 -> OFFSET_COMMIT_VALUE_SCHEMA_V1)
-  private val CURRENT_OFFSET_VALUE_SCHEMA_VERSION = 1.toShort
+    1 -> OFFSET_COMMIT_VALUE_SCHEMA_V1,
+    2 -> OFFSET_COMMIT_VALUE_SCHEMA_V2)
 
   // map of version of group metadata value schemas
   private val GROUP_VALUE_SCHEMAS = Map(
     0 -> GROUP_METADATA_VALUE_SCHEMA_V0,
-    1 -> GROUP_METADATA_VALUE_SCHEMA_V1)
-  private val CURRENT_GROUP_VALUE_SCHEMA_VERSION = 1.toShort
+    1 -> GROUP_METADATA_VALUE_SCHEMA_V1,
+    2 -> GROUP_METADATA_VALUE_SCHEMA_V2)
+  private val CURRENT_GROUP_VALUE_SCHEMA_VERSION = 2.toShort
 
   private val CURRENT_OFFSET_KEY_SCHEMA = schemaForKey(CURRENT_OFFSET_KEY_SCHEMA_VERSION)
   private val CURRENT_GROUP_KEY_SCHEMA = schemaForKey(CURRENT_GROUP_KEY_SCHEMA_VERSION)
 
-  private val CURRENT_OFFSET_VALUE_SCHEMA = schemaForOffset(CURRENT_OFFSET_VALUE_SCHEMA_VERSION)
+  private val CURRENT_OFFSET_VALUE_SCHEMA = schemaForOffset(2)
   private val CURRENT_GROUP_VALUE_SCHEMA = schemaForGroup(CURRENT_GROUP_VALUE_SCHEMA_VERSION)
 
   private def schemaForKey(version: Int) = {
@@ -1081,17 +1083,34 @@ object GroupMetadataManager {
    * Generates the payload for offset commit message from given offset and metadata
    *
    * @param offsetAndMetadata consumer's current offset and metadata
+   * @param apiVersion the api version
    * @return payload for offset commit message
    */
-  private[group] def offsetCommitValue(offsetAndMetadata: OffsetAndMetadata): Array[Byte] = {
-    // generate commit value with schema version 1
-    val value = new Struct(CURRENT_OFFSET_VALUE_SCHEMA)
-    value.set(OFFSET_VALUE_OFFSET_FIELD_V1, offsetAndMetadata.offset)
-    value.set(OFFSET_VALUE_METADATA_FIELD_V1, offsetAndMetadata.metadata)
-    value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1, offsetAndMetadata.commitTimestamp)
-    value.set(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1, offsetAndMetadata.expireTimestamp)
+  private[group] def offsetCommitValue(offsetAndMetadata: OffsetAndMetadata,
+                                       apiVersion: ApiVersion): Array[Byte] = {
+    // generate commit value according to schema version
+    val (version, value) = {
+      if (apiVersion < KAFKA_2_1_IV0 || offsetAndMetadata.expireTimestamp.nonEmpty)
+        // if an older version of the API is used, or if an explicit expiration is provided, use the older schema
+        (1.toShort, new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V1))
+      else
+        (2.toShort, new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V2))
+    }
+
+    if (version == 2) {
+      value.set(OFFSET_VALUE_OFFSET_FIELD_V2, offsetAndMetadata.offset)
+      value.set(OFFSET_VALUE_METADATA_FIELD_V2, offsetAndMetadata.metadata)
+      value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2, offsetAndMetadata.commitTimestamp)
+    } else {
+      value.set(OFFSET_VALUE_OFFSET_FIELD_V1, offsetAndMetadata.offset)
+      value.set(OFFSET_VALUE_METADATA_FIELD_V1, offsetAndMetadata.metadata)
+      value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1, offsetAndMetadata.commitTimestamp)
+      // version 1 has a non empty expireTimestamp field
+      value.set(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1, offsetAndMetadata.expireTimestamp.get)
+    }
+
     val byteBuffer = ByteBuffer.allocate(2 /* version */ + value.sizeOf)
-    byteBuffer.putShort(CURRENT_OFFSET_VALUE_SCHEMA_VERSION)
+    byteBuffer.putShort(version)
     value.writeTo(byteBuffer)
     byteBuffer.array()
   }
@@ -1102,19 +1121,30 @@ object GroupMetadataManager {
    *
    * @param groupMetadata current group metadata
    * @param assignment the assignment for the rebalancing generation
-   * @param version the version of the value message to use
+   * @param apiVersion the api version
    * @return payload for offset commit message
    */
   private[group] def groupMetadataValue(groupMetadata: GroupMetadata,
                                         assignment: Map[String, Array[Byte]],
-                                        version: Short = 0): Array[Byte] = {
-    val value = if (version == 0) new Struct(GROUP_METADATA_VALUE_SCHEMA_V0) else new Struct(CURRENT_GROUP_VALUE_SCHEMA)
+                                        apiVersion: ApiVersion): Array[Byte] = {
+
+    val (version, value) = {
+      if (apiVersion < KAFKA_0_10_1_IV0)
+        (0.toShort, new Struct(GROUP_METADATA_VALUE_SCHEMA_V0))
+      else if (apiVersion < KAFKA_2_1_IV0)
+        (1.toShort, new Struct(GROUP_METADATA_VALUE_SCHEMA_V1))
+      else
+        (2.toShort, new Struct(CURRENT_GROUP_VALUE_SCHEMA))
+    }
 
     value.set(PROTOCOL_TYPE_KEY, groupMetadata.protocolType.getOrElse(""))
     value.set(GENERATION_KEY, groupMetadata.generationId)
     value.set(PROTOCOL_KEY, groupMetadata.protocolOrNull)
     value.set(LEADER_KEY, groupMetadata.leaderOrNull)
 
+    if (version >= 2)
+      value.set(CURRENT_STATE_TIMESTAMP_KEY, groupMetadata.currentStateTimestampOrDefault)
+
     val memberArray = groupMetadata.allMemberMetadata.map { memberMetadata =>
       val memberStruct = value.instance(MEMBERS_KEY)
       memberStruct.set(MEMBER_ID_KEY, memberMetadata.memberId)
@@ -1174,7 +1204,7 @@ object GroupMetadataManager {
 
       GroupMetadataKey(version, group)
     } else {
-      throw new IllegalStateException("Unknown version " + version + " for group metadata message")
+      throw new IllegalStateException(s"Unknown group metadata message version: $version")
     }
   }
 
@@ -1205,8 +1235,14 @@ object GroupMetadataManager {
         val expireTimestamp = value.get(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1).asInstanceOf[Long]
 
         OffsetAndMetadata(offset, metadata, commitTimestamp, expireTimestamp)
+      } else if (version == 2) {
+        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V2).asInstanceOf[Long]
+        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V2).asInstanceOf[String]
+        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2).asInstanceOf[Long]
+
+        OffsetAndMetadata(offset, metadata, commitTimestamp)
       } else {
-        throw new IllegalStateException("Unknown offset message version")
+        throw new IllegalStateException(s"Unknown offset message version: $version")
       }
     }
   }
@@ -1215,9 +1251,10 @@ object GroupMetadataManager {
    * Decodes the group metadata messages' payload and retrieves its member metadata from it
    *
    * @param buffer input byte-buffer
+   * @param time the time instance to use
    * @return a group metadata object from the message
    */
-  def readGroupMessageValue(groupId: String, buffer: ByteBuffer): GroupMetadata = {
+  def readGroupMessageValue(groupId: String, buffer: ByteBuffer, time: Time): GroupMetadata = {
     if (buffer == null) { // tombstone
       null
     } else {
@@ -1225,13 +1262,23 @@ object GroupMetadataManager {
       val valueSchema = schemaForGroup(version)
       val value = valueSchema.read(buffer)
 
-      if (version == 0 || version == 1) {
+      if (version >= 0 && version <= 2) {
         val generationId = value.get(GENERATION_KEY).asInstanceOf[Int]
         val protocolType = value.get(PROTOCOL_TYPE_KEY).asInstanceOf[String]
         val protocol = value.get(PROTOCOL_KEY).asInstanceOf[String]
         val leaderId = value.get(LEADER_KEY).asInstanceOf[String]
         val memberMetadataArray = value.getArray(MEMBERS_KEY)
         val initialState = if (memberMetadataArray.isEmpty) Empty else Stable
+        val currentStateTimestamp: Option[Long] = version match {
+          case version if version == 2 =>
+            if (value.hasField(CURRENT_STATE_TIMESTAMP_KEY)) {
+              val timestamp = value.getLong(CURRENT_STATE_TIMESTAMP_KEY)
+              if (timestamp == -1) None else Some(timestamp)
+            } else
+              None
+          case _ =>
+            None
+        }
 
         val members = memberMetadataArray.map { memberMetadataObj =>
           val memberMetadata = memberMetadataObj.asInstanceOf[Struct]
@@ -1247,9 +1294,9 @@ object GroupMetadataManager {
           member.assignment = Utils.toArray(memberMetadata.get(ASSIGNMENT_KEY).asInstanceOf[ByteBuffer])
           member
         }
-        GroupMetadata.loadGroup(groupId, initialState, generationId, protocolType, protocol, leaderId, members)
+        GroupMetadata.loadGroup(groupId, initialState, generationId, protocolType, protocol, leaderId, currentStateTimestamp, members, time)
       } else {
-        throw new IllegalStateException("Unknown group metadata message version")
+        throw new IllegalStateException(s"Unknown group metadata message version: $version")
       }
     }
   }
@@ -1287,7 +1334,7 @@ object GroupMetadataManager {
           val value = consumerRecord.value
           val formattedValue =
             if (value == null) "NULL"
-            else GroupMetadataManager.readGroupMessageValue(groupId, ByteBuffer.wrap(value)).toString
+            else GroupMetadataManager.readGroupMessageValue(groupId, ByteBuffer.wrap(value), Time.SYSTEM).toString
           output.write(groupId.getBytes(StandardCharsets.UTF_8))
           output.write("::".getBytes(StandardCharsets.UTF_8))
           output.write(formattedValue.getBytes(StandardCharsets.UTF_8))
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 0c88be9bb5c42..de9e0bf66efff 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -332,33 +332,25 @@ class KafkaApis(val requestChannel: RequestChannel,
       } else {
         // for version 1 and beyond store offsets in offset manager
 
-        // compute the retention time based on the request version:
-        // if it is v1 or not specified by user, we can use the default retention
-        val offsetRetention =
-          if (header.apiVersion <= 1 ||
-            offsetCommitRequest.retentionTime == OffsetCommitRequest.DEFAULT_RETENTION_TIME)
-            groupCoordinator.offsetConfig.offsetsRetentionMs
-          else
-            offsetCommitRequest.retentionTime
-
         // commit timestamp is always set to now.
         // "default" expiration timestamp is now + retention (and retention may be overridden if v2)
         // expire timestamp is computed differently for v1 and v2.
-        //   - If v1 and no explicit commit timestamp is provided we use default expiration timestamp.
+        //   - If v1 and no explicit commit timestamp is provided we treat it the same as v5.
         //   - If v1 and explicit commit timestamp is provided we calculate retention from that explicit commit timestamp
-        //   - If v2 we use the default expiration timestamp
+        //   - If v2/v3/v4 (no explicit commit timestamp) we treat it the same as v5.
+        //   - For v5 and beyond there is no per partition expiration timestamp, so this field is no longer in effect
         val currentTimestamp = time.milliseconds
-        val defaultExpireTimestamp = offsetRetention + currentTimestamp
         val partitionData = authorizedTopicRequestInfo.mapValues { partitionData =>
           val metadata = if (partitionData.metadata == null) OffsetMetadata.NoMetadata else partitionData.metadata
           new OffsetAndMetadata(
             offsetMetadata = OffsetMetadata(partitionData.offset, metadata),
-            commitTimestamp = currentTimestamp,
-            expireTimestamp = {
-              if (partitionData.timestamp == OffsetCommitRequest.DEFAULT_TIMESTAMP)
-                defaultExpireTimestamp
-              else
-                offsetRetention + partitionData.timestamp
+            commitTimestamp = partitionData.timestamp match {
+              case OffsetCommitRequest.DEFAULT_TIMESTAMP => currentTimestamp
+              case customTimestamp => customTimestamp
+            },
+            expireTimestamp = offsetCommitRequest.retentionTime match {
+              case OffsetCommitRequest.DEFAULT_RETENTION_TIME => None
+              case retentionTime => Some(currentTimestamp + retentionTime)
             }
           )
         }
@@ -1912,7 +1904,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       topicPartition -> new OffsetAndMetadata(
         offsetMetadata = OffsetMetadata(partitionData.offset, metadata),
         commitTimestamp = currentTimestamp,
-        expireTimestamp = defaultExpireTimestamp)
+        expireTimestamp = Some(defaultExpireTimestamp))
     }
   }
 
diff --git a/core/src/main/scala/kafka/tools/ConsoleConsumer.scala b/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
index c55f6c484c754..7e2c5644a3b4a 100755
--- a/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
+++ b/core/src/main/scala/kafka/tools/ConsoleConsumer.scala
@@ -368,6 +368,10 @@ object ConsoleConsumer extends Logging {
         consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, group)
       case None =>
         consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, s"console-consumer-${new Random().nextInt(100000)}")
+        // By default, avoid unnecessary expansion of the coordinator cache since
+        // the auto-generated group and its offsets is not intended to be used again
+        if (!consumerProps.containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG))
+          consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false")
         groupIdPassed = false
     }
 
diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala
index 17fbd8f0f1430..1792c7bff8152 100755
--- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala
+++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala
@@ -29,7 +29,7 @@ import kafka.utils._
 import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
 import org.apache.kafka.common.KafkaException
 import org.apache.kafka.common.record._
-import org.apache.kafka.common.utils.Utils
+import org.apache.kafka.common.utils.{Time, Utils}
 
 import scala.collection.{Map, mutable}
 import scala.collection.mutable.ArrayBuffer
@@ -321,7 +321,7 @@ object DumpLogSegments {
 
     private def parseGroupMetadata(groupMetadataKey: GroupMetadataKey, payload: ByteBuffer) = {
       val groupId = groupMetadataKey.key
-      val group = GroupMetadataManager.readGroupMessageValue(groupId, payload)
+      val group = GroupMetadataManager.readGroupMessageValue(groupId, payload, Time.SYSTEM)
       val protocolType = group.protocolType.getOrElse("")
 
       val assignment = group.allMemberMetadata.map { member =>
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index f4ff8e17d249d..69b65252b0237 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -334,7 +334,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
   private def createOffsetCommitRequest = {
     new requests.OffsetCommitRequest.Builder(
       group, Map(tp -> new requests.OffsetCommitRequest.PartitionData(0, "metadata")).asJava).
-      setMemberId("").setGenerationId(1).setRetentionTime(1000).
+      setMemberId("").setGenerationId(1).
       build()
   }
 
diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
index 32e40850fcfaf..2befc8fdf3b0b 100644
--- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
@@ -79,6 +79,9 @@ class ApiVersionTest {
     assertEquals(KAFKA_2_0_IV1, ApiVersion("2.0"))
     assertEquals(KAFKA_2_0_IV0, ApiVersion("2.0-IV0"))
     assertEquals(KAFKA_2_0_IV1, ApiVersion("2.0-IV1"))
+
+    assertEquals(KAFKA_2_1_IV0, ApiVersion("2.1"))
+    assertEquals(KAFKA_2_1_IV0, ApiVersion("2.1-IV0"))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index 3bfacabc843a0..77e6fdc36830e 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -17,20 +17,22 @@
 
 package kafka.coordinator.group
 
-import kafka.api.ApiVersion
+import kafka.api.{ApiVersion, KAFKA_1_1_IV0, KAFKA_2_1_IV0}
 import kafka.cluster.Partition
 import kafka.common.OffsetAndMetadata
 import kafka.log.{Log, LogAppendInfo}
 import kafka.server.{FetchDataInfo, KafkaConfig, LogOffsetMetadata, ReplicaManager}
 import kafka.utils.TestUtils.fail
 import kafka.utils.{KafkaScheduler, MockTime, TestUtils}
+import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
+import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.{IsolationLevel, OffsetFetchResponse}
 import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
 import org.easymock.{Capture, EasyMock, IAnswer}
-import org.junit.Assert.{assertEquals, assertFalse, assertTrue, assertNull}
+import org.junit.Assert.{assertEquals, assertFalse, assertNull, assertTrue}
 import org.junit.{Before, Test}
 import java.nio.ByteBuffer
 
@@ -52,6 +54,7 @@ class GroupMetadataManagerTest {
   var scheduler: KafkaScheduler = null
   var zkClient: KafkaZkClient = null
   var partition: Partition = null
+  var defaultOffsetRetentionMs = Long.MaxValue
 
   val groupId = "foo"
   val groupPartitionId = 0
@@ -75,6 +78,8 @@ class GroupMetadataManagerTest {
       offsetCommitTimeoutMs = config.offsetCommitTimeoutMs,
       offsetCommitRequiredAcks = config.offsetCommitRequiredAcks)
 
+    defaultOffsetRetentionMs = offsetConfig.offsetsRetentionMs
+
     // make two partitions of the group topic to make sure some partitions are not owned by the coordinator
     zkClient = EasyMock.createNiceMock(classOf[KafkaZkClient])
     EasyMock.expect(zkClient.getTopicPartitionCount(Topic.GROUP_METADATA_TOPIC_NAME)).andReturn(Some(2))
@@ -506,7 +511,7 @@ class GroupMetadataManagerTest {
     // group is owned but does not exist yet
     assertTrue(groupMetadataManager.groupNotExists(groupId))
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     // group is owned but not Dead
@@ -616,6 +621,7 @@ class GroupMetadataManagerTest {
     assertEquals(committedOffsets.size, group.allOffsets.size)
     committedOffsets.foreach { case (topicPartition, offset) =>
       assertEquals(Some(offset), group.offset(topicPartition).map(_.offset))
+      assertTrue(group.offset(topicPartition).map(_.expireTimestamp).contains(None))
     }
   }
 
@@ -729,9 +735,9 @@ class GroupMetadataManagerTest {
 
   @Test
   def testAddGroup() {
-    val group = new GroupMetadata("foo", initialState = Empty)
+    val group = new GroupMetadata("foo", Empty, time)
     assertEquals(group, groupMetadataManager.addGroup(group))
-    assertEquals(group, groupMetadataManager.addGroup(new GroupMetadata("foo", initialState = Empty)))
+    assertEquals(group, groupMetadataManager.addGroup(new GroupMetadata("foo", Empty, time)))
   }
 
   @Test
@@ -739,7 +745,7 @@ class GroupMetadataManagerTest {
     val generation = 27
     val protocolType = "consumer"
 
-    val group = GroupMetadata.loadGroup(groupId, Empty, generation, protocolType, null, null, Seq.empty)
+    val group = GroupMetadata.loadGroup(groupId, Empty, generation, protocolType, null, null, None, Seq.empty, time)
     groupMetadataManager.addGroup(group)
 
     val capturedRecords = expectAppendMessage(Errors.NONE)
@@ -758,7 +764,7 @@ class GroupMetadataManagerTest {
     assertEquals(1, records.size)
 
     val record = records.head
-    val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value)
+    val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value, time)
     assertTrue(groupMetadata.is(Empty))
     assertEquals(generation, groupMetadata.generationId)
     assertEquals(Some(protocolType), groupMetadata.protocolType)
@@ -766,7 +772,7 @@ class GroupMetadataManagerTest {
 
   @Test
   def testStoreEmptySimpleGroup() {
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val capturedRecords = expectAppendMessage(Errors.NONE)
@@ -787,7 +793,7 @@ class GroupMetadataManagerTest {
     assertEquals(1, records.size)
 
     val record = records.head
-    val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value)
+    val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value, time)
     assertTrue(groupMetadata.is(Empty))
     assertEquals(0, groupMetadata.generationId)
     assertEquals(None, groupMetadata.protocolType)
@@ -809,7 +815,7 @@ class GroupMetadataManagerTest {
   private def assertStoreGroupErrorMapping(appendError: Errors, expectedError: Errors) {
     EasyMock.reset(replicaManager)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     expectAppendMessage(appendError)
@@ -832,7 +838,7 @@ class GroupMetadataManagerTest {
     val clientId = "clientId"
     val clientHost = "localhost"
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val member = new MemberMetadata(memberId, groupId, clientId, clientHost, rebalanceTimeout, sessionTimeout,
@@ -863,7 +869,7 @@ class GroupMetadataManagerTest {
     val clientId = "clientId"
     val clientHost = "localhost"
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
 
     val member = new MemberMetadata(memberId, groupId, clientId, clientHost, rebalanceTimeout, sessionTimeout,
       protocolType, List(("protocol", Array[Byte]())))
@@ -893,7 +899,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val offsets = immutable.Map(topicPartition -> OffsetAndMetadata(offset))
@@ -935,7 +941,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val offsets = immutable.Map(topicPartition -> OffsetAndMetadata(offset))
@@ -975,7 +981,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val offsets = immutable.Map(topicPartition -> OffsetAndMetadata(offset))
@@ -1014,7 +1020,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val offsets = immutable.Map(topicPartition -> OffsetAndMetadata(offset))
@@ -1052,7 +1058,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val offsets = immutable.Map(topicPartition -> OffsetAndMetadata(offset))
@@ -1094,7 +1100,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     val offsets = immutable.Map(topicPartition -> OffsetAndMetadata(offset))
@@ -1132,7 +1138,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     // expire the offset after 1 millisecond
@@ -1185,7 +1191,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
     group.generationId = 5
 
@@ -1233,7 +1239,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
     group.generationId = 5
 
@@ -1287,7 +1293,7 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
     // expire the offset after 1 millisecond
@@ -1348,31 +1354,39 @@ class GroupMetadataManagerTest {
   }
 
   @Test
-  def testExpireOffsetsWithActiveGroup() {
+  def testOffsetExpirationSemantics() {
     val memberId = "memberId"
     val clientId = "clientId"
     val clientHost = "localhost"
-    val topicPartition1 = new TopicPartition("foo", 0)
-    val topicPartition2 = new TopicPartition("foo", 1)
+    val topic = "foo"
+    val topicPartition1 = new TopicPartition(topic, 0)
+    val topicPartition2 = new TopicPartition(topic, 1)
+    val topicPartition3 = new TopicPartition(topic, 2)
     val offset = 37
 
     groupMetadataManager.addPartitionOwnership(groupPartitionId)
 
-    val group = new GroupMetadata(groupId, initialState = Empty)
+    val group = new GroupMetadata(groupId, Empty, time)
     groupMetadataManager.addGroup(group)
 
+    val subscription = new Subscription(List(topic).asJava)
     val member = new MemberMetadata(memberId, groupId, clientId, clientHost, rebalanceTimeout, sessionTimeout,
-      protocolType, List(("protocol", Array[Byte]())))
+      protocolType, List(("protocol", ConsumerProtocol.serializeSubscription(subscription).array())))
     member.awaitingJoinCallback = _ => ()
     group.add(member)
     group.transitionTo(PreparingRebalance)
     group.initNextGeneration()
 
-    // expire the offset after 1 millisecond
     val startMs = time.milliseconds
+    // old clients, expiry timestamp is explicitly set
+    val tp1OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs, startMs + 1)
+    val tp2OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs, startMs + 3)
+    // new clients, no per-partition expiry timestamp, offsets of group expire together
+    val tp3OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
     val offsets = immutable.Map(
-      topicPartition1 -> OffsetAndMetadata(offset, "", startMs, startMs + 1),
-      topicPartition2 -> OffsetAndMetadata(offset, "", startMs, startMs + 3))
+      topicPartition1 -> tp1OffsetAndMetadata,
+      topicPartition2 -> tp2OffsetAndMetadata,
+      topicPartition3 -> tp3OffsetAndMetadata)
 
     mockGetPartition()
     expectAppendMessage(Errors.NONE)
@@ -1389,8 +1403,26 @@ class GroupMetadataManagerTest {
     assertFalse(commitErrors.isEmpty)
     assertEquals(Some(Errors.NONE), commitErrors.get.get(topicPartition1))
 
-    // expire all of the offsets
-    time.sleep(4)
+    // do not expire any offset even though expiration timestamp is reached for one (due to group still being active)
+    time.sleep(2)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    // group and offsets should still be there
+    assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
+    assertEquals(Some(tp1OffsetAndMetadata), group.offset(topicPartition1))
+    assertEquals(Some(tp2OffsetAndMetadata), group.offset(topicPartition2))
+    assertEquals(Some(tp3OffsetAndMetadata), group.offset(topicPartition3))
+
+    var cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1, topicPartition2, topicPartition3)))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition1).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition2).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition3).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    group.transitionTo(PreparingRebalance)
+    group.transitionTo(Empty)
 
     // expect the offset tombstone
     EasyMock.reset(partition)
@@ -1401,16 +1433,245 @@ class GroupMetadataManagerTest {
 
     groupMetadataManager.cleanupGroupMetadata()
 
-    // group should still be there, but the offsets should be gone
+    // group is empty now, only one offset should expire
+    assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
+    assertEquals(None, group.offset(topicPartition1))
+    assertEquals(Some(tp2OffsetAndMetadata), group.offset(topicPartition2))
+    assertEquals(Some(tp3OffsetAndMetadata), group.offset(topicPartition3))
+
+    cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1, topicPartition2, topicPartition3)))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition1).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition2).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition3).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    time.sleep(2)
+
+    // expect the offset tombstone
+    EasyMock.reset(partition)
+    EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
+      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      .andReturn(LogAppendInfo.UnknownLogAppendInfo)
+    EasyMock.replay(partition)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    // one more offset should expire
     assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
     assertEquals(None, group.offset(topicPartition1))
     assertEquals(None, group.offset(topicPartition2))
+    assertEquals(Some(tp3OffsetAndMetadata), group.offset(topicPartition3))
 
-    val cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1, topicPartition2)))
+    cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1, topicPartition2, topicPartition3)))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition1).map(_.offset))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition2).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition3).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    // advance time to just before the offset of last partition is to be expired, no offset should expire
+    time.sleep(group.currentStateTimestamp.get + defaultOffsetRetentionMs - time.milliseconds() - 1)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    // one more offset should expire
+    assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
+    assertEquals(None, group.offset(topicPartition1))
+    assertEquals(None, group.offset(topicPartition2))
+    assertEquals(Some(tp3OffsetAndMetadata), group.offset(topicPartition3))
+
+    cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1, topicPartition2, topicPartition3)))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition1).map(_.offset))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition2).map(_.offset))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition3).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    // advance time enough for that last offset to expire
+    time.sleep(2)
+
+    // expect the offset tombstone
+    EasyMock.reset(partition)
+    EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
+      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      .andReturn(LogAppendInfo.UnknownLogAppendInfo)
+    EasyMock.replay(partition)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    // group and all its offsets should be gone now
+    assertEquals(None, groupMetadataManager.getGroup(groupId))
+    assertEquals(None, group.offset(topicPartition1))
+    assertEquals(None, group.offset(topicPartition2))
+    assertEquals(None, group.offset(topicPartition3))
+
+    cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1, topicPartition2, topicPartition3)))
     assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition1).map(_.offset))
     assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition2).map(_.offset))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition3).map(_.offset))
 
     EasyMock.verify(replicaManager)
+
+    assert(group.is(Dead))
+  }
+
+  @Test
+  def testOffsetExpirationOfSimpleConsumer() {
+    val memberId = "memberId"
+    val clientId = "clientId"
+    val clientHost = "localhost"
+    val topic = "foo"
+    val topicPartition1 = new TopicPartition(topic, 0)
+    val offset = 37
+
+    groupMetadataManager.addPartitionOwnership(groupPartitionId)
+
+    val group = new GroupMetadata(groupId, Empty, time)
+    groupMetadataManager.addGroup(group)
+
+    // expire the offset after 1 and 3 milliseconds (old clients) and after default retention (new clients)
+    val startMs = time.milliseconds
+    // old clients, expiry timestamp is explicitly set
+    val tp1OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
+    val tp2OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs)
+    // new clients, no per-partition expiry timestamp, offsets of group expire together
+    val offsets = immutable.Map(
+      topicPartition1 -> tp1OffsetAndMetadata)
+
+    mockGetPartition()
+    expectAppendMessage(Errors.NONE)
+    EasyMock.replay(replicaManager)
+
+    var commitErrors: Option[immutable.Map[TopicPartition, Errors]] = None
+    def callback(errors: immutable.Map[TopicPartition, Errors]) {
+      commitErrors = Some(errors)
+    }
+
+    groupMetadataManager.storeOffsets(group, memberId, offsets, callback)
+    assertTrue(group.hasOffsets)
+
+    assertFalse(commitErrors.isEmpty)
+    assertEquals(Some(Errors.NONE), commitErrors.get.get(topicPartition1))
+
+    // do not expire offsets while within retention period since commit timestamp
+    val expiryTimestamp = offsets.get(topicPartition1).get.commitTimestamp + defaultOffsetRetentionMs
+    time.sleep(expiryTimestamp - time.milliseconds() - 1)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    // group and offsets should still be there
+    assertEquals(Some(group), groupMetadataManager.getGroup(groupId))
+    assertEquals(Some(tp1OffsetAndMetadata), group.offset(topicPartition1))
+
+    var cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1)))
+    assertEquals(Some(offset), cachedOffsets.get(topicPartition1).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    // advance time to enough for offsets to expire
+    time.sleep(2)
+
+    // expect the offset tombstone
+    EasyMock.reset(partition)
+    EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]),
+      isFromClient = EasyMock.eq(false), requiredAcks = EasyMock.anyInt()))
+      .andReturn(LogAppendInfo.UnknownLogAppendInfo)
+    EasyMock.replay(partition)
+
+    groupMetadataManager.cleanupGroupMetadata()
+
+    // group and all its offsets should be gone now
+    assertEquals(None, groupMetadataManager.getGroup(groupId))
+    assertEquals(None, group.offset(topicPartition1))
+
+    cachedOffsets = groupMetadataManager.getOffsets(groupId, Some(Seq(topicPartition1)))
+    assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicPartition1).map(_.offset))
+
+    EasyMock.verify(replicaManager)
+
+    assert(group.is(Dead))
+  }
+
+  @Test
+  def testLoadOffsetFromOldCommit() = {
+    val groupMetadataTopicPartition = groupTopicPartition
+    val generation = 935
+    val protocolType = "consumer"
+    val protocol = "range"
+    val startOffset = 15L
+    val committedOffsets = Map(
+      new TopicPartition("foo", 0) -> 23L,
+      new TopicPartition("foo", 1) -> 455L,
+      new TopicPartition("bar", 0) -> 8992L
+    )
+
+    val apiVersion = KAFKA_1_1_IV0
+    val offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, apiVersion = apiVersion, retentionTime = Some(100))
+    val memberId = "98098230493"
+    val groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId, apiVersion)
+    val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE,
+      offsetCommitRecords ++ Seq(groupMetadataRecord): _*)
+
+    expectGroupMetadataLoad(groupMetadataTopicPartition, startOffset, records)
+
+    EasyMock.replay(replicaManager)
+
+    groupMetadataManager.loadGroupsAndOffsets(groupMetadataTopicPartition, _ => ())
+
+    val group = groupMetadataManager.getGroup(groupId).getOrElse(fail("Group was not loaded into the cache"))
+    assertEquals(groupId, group.groupId)
+    assertEquals(Stable, group.currentState)
+    assertEquals(memberId, group.leaderOrNull)
+    assertEquals(generation, group.generationId)
+    assertEquals(Some(protocolType), group.protocolType)
+    assertEquals(protocol, group.protocolOrNull)
+    assertEquals(Set(memberId), group.allMembers)
+    assertEquals(committedOffsets.size, group.allOffsets.size)
+    committedOffsets.foreach { case (topicPartition, offset) =>
+      assertEquals(Some(offset), group.offset(topicPartition).map(_.offset))
+      assertTrue(group.offset(topicPartition).map(_.expireTimestamp).get.nonEmpty)
+    }
+  }
+
+  @Test
+  def testLoadOffsetWithExplicitRetention() = {
+    val groupMetadataTopicPartition = groupTopicPartition
+    val generation = 935
+    val protocolType = "consumer"
+    val protocol = "range"
+    val startOffset = 15L
+    val committedOffsets = Map(
+      new TopicPartition("foo", 0) -> 23L,
+      new TopicPartition("foo", 1) -> 455L,
+      new TopicPartition("bar", 0) -> 8992L
+    )
+
+    val offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, retentionTime = Some(100))
+    val memberId = "98098230493"
+    val groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId)
+    val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE,
+      offsetCommitRecords ++ Seq(groupMetadataRecord): _*)
+
+    expectGroupMetadataLoad(groupMetadataTopicPartition, startOffset, records)
+
+    EasyMock.replay(replicaManager)
+
+    groupMetadataManager.loadGroupsAndOffsets(groupMetadataTopicPartition, _ => ())
+
+    val group = groupMetadataManager.getGroup(groupId).getOrElse(fail("Group was not loaded into the cache"))
+    assertEquals(groupId, group.groupId)
+    assertEquals(Stable, group.currentState)
+    assertEquals(memberId, group.leaderOrNull)
+    assertEquals(generation, group.generationId)
+    assertEquals(Some(protocolType), group.protocolType)
+    assertEquals(protocol, group.protocolOrNull)
+    assertEquals(Set(memberId), group.allMembers)
+    assertEquals(committedOffsets.size, group.allOffsets.size)
+    committedOffsets.foreach { case (topicPartition, offset) =>
+      assertEquals(Some(offset), group.offset(topicPartition).map(_.offset))
+      assertTrue(group.offset(topicPartition).map(_.expireTimestamp).get.nonEmpty)
+    }
   }
 
   private def appendAndCaptureCallback(): Capture[Map[TopicPartition, PartitionResponse] => Unit] = {
@@ -1452,20 +1713,21 @@ class GroupMetadataManagerTest {
   private def buildStableGroupRecordWithMember(generation: Int,
                                                protocolType: String,
                                                protocol: String,
-                                               memberId: String): SimpleRecord = {
+                                               memberId: String,
+                                               apiVersion: ApiVersion = ApiVersion.latestVersion): SimpleRecord = {
     val memberProtocols = List((protocol, Array.emptyByteArray))
     val member = new MemberMetadata(memberId, groupId, "clientId", "clientHost", 30000, 10000, protocolType, memberProtocols)
-    val group = GroupMetadata.loadGroup(groupId, Stable, generation, protocolType, protocol,
-      leaderId = memberId, Seq(member))
+    val group = GroupMetadata.loadGroup(groupId, Stable, generation, protocolType, protocol, memberId,
+      if (apiVersion >= KAFKA_2_1_IV0) Some(time.milliseconds()) else None, Seq(member), time)
     val groupMetadataKey = GroupMetadataManager.groupMetadataKey(groupId)
-    val groupMetadataValue = GroupMetadataManager.groupMetadataValue(group, Map(memberId -> Array.empty[Byte]))
+    val groupMetadataValue = GroupMetadataManager.groupMetadataValue(group, Map(memberId -> Array.empty[Byte]), apiVersion)
     new SimpleRecord(groupMetadataKey, groupMetadataValue)
   }
 
   private def buildEmptyGroupRecord(generation: Int, protocolType: String): SimpleRecord = {
-    val group = GroupMetadata.loadGroup(groupId, Empty, generation, protocolType, null, null, Seq.empty)
+    val group = GroupMetadata.loadGroup(groupId, Empty, generation, protocolType, null, null, None, Seq.empty, time)
     val groupMetadataKey = GroupMetadataManager.groupMetadataKey(groupId)
-    val groupMetadataValue = GroupMetadataManager.groupMetadataValue(group, Map.empty)
+    val groupMetadataValue = GroupMetadataManager.groupMetadataValue(group, Map.empty, ApiVersion.latestVersion)
     new SimpleRecord(groupMetadataKey, groupMetadataValue)
   }
 
@@ -1511,11 +1773,19 @@ class GroupMetadataManagerTest {
   }
 
   private def createCommittedOffsetRecords(committedOffsets: Map[TopicPartition, Long],
-                                           groupId: String = groupId): Seq[SimpleRecord] = {
+                                           groupId: String = groupId,
+                                           apiVersion: ApiVersion = ApiVersion.latestVersion,
+                                           retentionTime: Option[Long] = None): Seq[SimpleRecord] = {
     committedOffsets.map { case (topicPartition, offset) =>
-      val offsetAndMetadata = OffsetAndMetadata(offset)
+      val offsetAndMetadata = retentionTime match {
+        case Some(timestamp) =>
+          val commitTimestamp = time.milliseconds()
+          OffsetAndMetadata(offset, "", commitTimestamp, commitTimestamp + timestamp)
+        case None =>
+          OffsetAndMetadata(offset)
+      }
       val offsetCommitKey = GroupMetadataManager.offsetCommitKey(groupId, topicPartition)
-      val offsetCommitValue = GroupMetadataManager.offsetCommitValue(offsetAndMetadata)
+      val offsetCommitValue = GroupMetadataManager.offsetCommitValue(offsetAndMetadata, apiVersion)
       new SimpleRecord(offsetCommitKey, offsetCommitValue)
     }.toSeq
   }
@@ -1542,7 +1812,7 @@ class GroupMetadataManagerTest {
   def testMetrics() {
     groupMetadataManager.cleanupGroupMetadata()
     expectMetrics(groupMetadataManager, 0, 0, 0)
-    val group = new GroupMetadata("foo2", Stable)
+    val group = new GroupMetadata("foo2", Stable, time)
     groupMetadataManager.addGroup(group)
     expectMetrics(groupMetadataManager, 1, 0, 0)
     group.transitionTo(PreparingRebalance)
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
index 183860f215535..90545339ca58f 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
@@ -18,9 +18,8 @@
 package kafka.coordinator.group
 
 import kafka.common.OffsetAndMetadata
-
 import org.apache.kafka.common.TopicPartition
-
+import org.apache.kafka.common.utils.Time
 import org.junit.Assert._
 import org.junit.{Before, Test}
 import org.scalatest.junit.JUnitSuite
@@ -40,7 +39,7 @@ class GroupMetadataTest extends JUnitSuite {
 
   @Before
   def setUp() {
-    group = new GroupMetadata("groupId", initialState = Empty)
+    group = new GroupMetadata("groupId", Empty, Time.SYSTEM)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 0205bcf214156..d91e008fe3bcc 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -240,7 +240,7 @@ class RequestQuotaTest extends BaseRequestTest {
         case ApiKeys.OFFSET_COMMIT =>
           new OffsetCommitRequest.Builder("test-group",
             Map(tp -> new OffsetCommitRequest.PartitionData(0, "metadata")).asJava).
-            setMemberId("").setGenerationId(1).setRetentionTime(1000)
+            setMemberId("").setGenerationId(1)
 
         case ApiKeys.OFFSET_FETCH =>
           new OffsetFetchRequest.Builder("test-group", List(tp).asJava)
diff --git a/docs/upgrade.html b/docs/upgrade.html
index cb246f60b150e..26c0d15e978b4 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -19,6 +19,18 @@
 
 
 
 
    
diff --git a/docs/configuration.html b/docs/configuration.html
index e5576f9263beb..bde53a749c753 100644
--- a/docs/configuration.html
+++ b/docs/configuration.html
@@ -260,190 +260,14 @@ 
    3.2 Topic-Level Configs
    
 
   
    3.3 Producer Configs
    
 
-  Below is the configuration of the Java producer:
+  Below is the configuration of the producer:
   
 
-  
    
-      For those interested in the legacy Scala producer configs, information can be found 
-      here.
-  
    
-
   
    3.4 Consumer Configs
    
 
-  In 0.9.0.0 we introduced the new Java consumer as a replacement for the older Scala-based simple and high-level consumers.
-  The configs for both new and old consumers are described below.
-
-  
    3.4.1 New Consumer Configs
    
-  Below is the configuration for the new consumer:
+  Below is the configuration for the consumer:
   
 
-  
    3.4.2 Old Consumer Configs
    
-
-  The essential old consumer configurations are the following:
-  
      
-          
    • group.id
-          
    • zookeeper.connect
-  
    
-
-  
    







                         
                         
    Join operands
                             Type
 
    
-  
-          
-          
-          
-  
-      
-        
-        
-        
-      
-      
-        
-        
-            
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-      
-        
-        
-        
-      
-  
-  
    PropertyDefaultDescription
    group.idA string that uniquely identifies the group of consumer processes to which this consumer belongs. By setting the same group id multiple processes indicate that they are all part of the same consumer group.
    zookeeper.connectSpecifies the ZooKeeper connection string in the form hostname:port where host and port are the host and port of a ZooKeeper server. To allow connecting through other ZooKeeper nodes when that ZooKeeper machine is down you can also specify multiple hosts in the form hostname1:port1,hostname2:port2,hostname3:port3.
-          
    
-      The server may also have a ZooKeeper chroot path as part of its ZooKeeper connection string which puts its data under some path in the global ZooKeeper namespace. If so the consumer should use the same chroot path in its connection string. For example to give a chroot path of /chroot/path you would give the connection string as  hostname1:port1,hostname2:port2,hostname3:port3/chroot/path.
    consumer.idnull
-          
    Generated automatically if not set.
    
-      
    socket.timeout.ms30 * 1000The socket timeout for network requests. The actual timeout set will be fetch.wait.max.ms + socket.timeout.ms.
    socket.receive.buffer.bytes64 * 1024The socket receive buffer for network requests
    fetch.message.max.bytes1024 * 1024The number of bytes of messages to attempt to fetch for each topic-partition in each fetch request. These bytes will be read into memory for each partition, so this helps control the memory used by the consumer. The fetch request size must be at least as large as the maximum message size the server allows or else it is possible for the producer to send messages larger than the consumer can fetch.
    num.consumer.fetchers1The number fetcher threads used to fetch data.
    auto.commit.enabletrueIf true, periodically commit to ZooKeeper the offset of messages already fetched by the consumer. This committed offset will be used when the process fails as the position from which the new consumer will begin.
    auto.commit.interval.ms60 * 1000The frequency in ms that the consumer offsets are committed to zookeeper.
    queued.max.message.chunks2Max number of message chunks buffered for consumption. Each chunk can be up to fetch.message.max.bytes.
    rebalance.max.retries4When a new consumer joins a consumer group the set of consumers attempt to "rebalance" the load to assign partitions to each consumer. If the set of consumers changes while this assignment is taking place the rebalance will fail and retry. This setting controls the maximum number of attempts before giving up.
    fetch.min.bytes1The minimum amount of data the server should return for a fetch request. If insufficient data is available the request will wait for that much data to accumulate before answering the request.
    fetch.wait.max.ms100The maximum amount of time the server will block before answering the fetch request if there isn't sufficient data to immediately satisfy fetch.min.bytes
    rebalance.backoff.ms2000Backoff time between retries during rebalance. If not set explicitly, the value in zookeeper.sync.time.ms is used.
-        
    refresh.leader.backoff.ms200Backoff time to wait before trying to determine the leader of a partition that has just lost its leader.
    auto.offset.resetlargest
-          
    What to do when there is no initial offset in ZooKeeper or if an offset is out of range:
    * smallest : automatically reset the offset to the smallest offset
    * largest : automatically reset the offset to the largest offset
    * anything else: throw exception to the consumer
    
-      
    consumer.timeout.ms-1Throw a timeout exception to the consumer if no message is available for consumption after the specified interval
    exclude.internal.topicstrueWhether messages from internal topics (such as offsets) should be exposed to the consumer.
    client.idgroup id valueThe client id is a user-specified string sent in each request to help trace calls. It should logically identify the application making the request.
    zookeeper.session.timeout.ms 6000ZooKeeper session timeout. If the consumer fails to heartbeat to ZooKeeper for this period of time it is considered dead and a rebalance will occur.
    zookeeper.connection.timeout.ms6000The max time that the client waits while establishing a connection to zookeeper.
    zookeeper.sync.time.ms 2000How far a ZK follower can be behind a ZK leader
    offsets.storagezookeeperSelect where offsets should be stored (zookeeper or kafka).
    offsets.channel.backoff.ms1000The backoff period when reconnecting the offsets channel or retrying failed offset fetch/commit requests.
    offsets.channel.socket.timeout.ms10000Socket timeout when reading responses for offset fetch/commit requests. This timeout is also used for ConsumerMetadata requests that are used to query for the offset manager.
    offsets.commit.max.retries5Retry the offset commit up to this many times on failure. This retry count only applies to offset commits during shut-down. It does not apply to commits originating from the auto-commit thread. It also does not apply to attempts to query for the offset coordinator before committing offsets. i.e., if a consumer metadata request fails for any reason, it will be retried and that retry does not count toward this limit.
    dual.commit.enabledtrueIf you are using "kafka" as offsets.storage, you can dual commit offsets to ZooKeeper (in addition to Kafka). This is required during migration from zookeeper-based offset storage to kafka-based offset storage. With respect to any given consumer group, it is safe to turn this off after all instances within that group have been migrated to the new version that commits offsets to the broker (instead of directly to ZooKeeper).
    partition.assignment.strategyrange
    Select between the "range" or "roundrobin" strategy for assigning partitions to consumer streams.
    The round-robin partition assignor lays out all the available partitions and all the available consumer threads. It then proceeds to do a round-robin assignment from partition to consumer thread. If the subscriptions of all consumer instances are identical, then the partitions will be uniformly distributed. (i.e., the partition ownership counts will be within a delta of exactly one across all consumer threads.) Round-robin assignment is permitted only if: (a) Every topic has the same number of streams within a consumer instance (b) The set of subscribed topics is identical for every consumer instance within the group.
     Range partitioning works on a per-topic basis. For each topic, we lay out the available partitions in numeric order and the consumer threads in lexicographic order. We then divide the number of partitions by the total number of consumer streams (threads) to determine the number of partitions to assign to each consumer. If it does not evenly divide, then the first few consumers will have one extra partition.
    
-
-
-  
    More details about consumer configuration can be found in the scala class kafka.consumer.ConsumerConfig.
    
-
   
    3.5 Kafka Connect Configs
    
   Below is the configuration of the Kafka Connect framework.
   
diff --git a/docs/implementation.html b/docs/implementation.html
index 8e1c50a68953b..4ecce7b448578 100644
--- a/docs/implementation.html
+++ b/docs/implementation.html
@@ -231,31 +231,29 @@ 
    Guarantees
    
     
    5.5 Distribution
    
     
    Consumer Offset Tracking
    
     
    
-    The high-level consumer tracks the maximum offset it has consumed in each partition and periodically commits its offset vector so that it can resume from those offsets in the event of a restart. Kafka provides the option to store all the offsets for a given consumer group in a designated broker (for that group) called the offset manager. i.e., any consumer instance in that consumer group should send its offset commits and fetches to that offset manager (broker). The high-level consumer handles this automatically. If you use the simple consumer you will need to manage offsets manually. This is currently unsupported in the Java simple consumer which can only commit or fetch offsets in ZooKeeper. If you use the Scala simple consumer you can discover the offset manager and explicitly commit or fetch offsets to the offset manager. A consumer can look up its offset manager by issuing a GroupCoordinatorRequest to any Kafka broker and reading the GroupCoordinatorResponse which will contain the offset manager. The consumer can then proceed to commit or fetch offsets from the offsets manager broker. In case the offset manager moves, the consumer will need to rediscover the offset manager. If you wish to manage your offsets manually, you can take a look at these code samples that explain how to issue OffsetCommitRequest and OffsetFetchRequest.
+    Kafka consumer tracks the maximum offset it has consumed in each partition and has the capability to commit offsets so
+        that it can resume from those offsets in the event of a restart. Kafka provides the option to store all the offsets for
+        a given consumer group in a designated broker (for that group) called the group coordinator.  i.e., any consumer instance
+        in that consumer group should send its offset commits and fetches to that group coordinator (broker). Consumer groups are
+        assigned to coordinators based on their group names. A consumer can look up its coordinator by issuing a FindCoordinatorRequest
+        to any Kafka broker and reading the FindCoordinatorResponse which will contain the coordinator details. The consumer
+        can then proceed to commit or fetch offsets from the coordinator broker. In case the coordinator moves, the consumer will
+        need to rediscover the coordinator. Offset commits can be done automatically or manually by consumer instance.
     
    
 
     
    
-    When the offset manager receives an OffsetCommitRequest, it appends the request to a special compacted Kafka topic named __consumer_offsets. The offset manager sends a successful offset commit response to the consumer only after all the replicas of the offsets topic receive the offsets. In case the offsets fail to replicate within a configurable timeout, the offset commit will fail and the consumer may retry the commit after backing off. (This is done automatically by the high-level consumer.) The brokers periodically compact the offsets topic since it only needs to maintain the most recent offset commit per partition. The offset manager also caches the offsets in an in-memory table in order to serve offset fetches quickly.
+    When the group coordinator receives an OffsetCommitRequest, it appends the request to a special compacted Kafka topic named __consumer_offsets.
+        The broker sends a successful offset commit response to the consumer only after all the replicas of the offsets topic receive the offsets.
+        In case the offsets fail to replicate within a configurable timeout, the offset commit will fail and the consumer may retry the commit after backing off.
+        The brokers periodically compact the offsets topic since it only needs to maintain the most recent offset commit per partition.
+        The coordinator also caches the offsets in an in-memory table in order to serve offset fetches quickly.
     
    
 
     
    
-    When the offset manager receives an offset fetch request, it simply returns the last committed offset vector from the offsets cache. In case the offset manager was just started or if it just became the offset manager for a new set of consumer groups (by becoming a leader for a partition of the offsets topic), it may need to load the offsets topic partition into the cache. In this case, the offset fetch will fail with an OffsetsLoadInProgress exception and the consumer may retry the OffsetFetchRequest after backing off. (This is done automatically by the high-level consumer.)
-    
    
-
-    
    Migrating offsets from ZooKeeper to Kafka
    
-    
    
-    Kafka consumers in earlier releases store their offsets by default in ZooKeeper. It is possible to migrate these consumers to commit offsets into Kafka by following these steps:
-    
      
-    
    1. Set offsets.storage=kafka and dual.commit.enabled=true in your consumer config.
-    
      2. 
-    
    2. Do a rolling bounce of your consumers and then verify that your consumers are healthy.
-    
      3. 
-    
    3. Set dual.commit.enabled=false in your consumer config.
-    
      4. 
-    
    4. Do a rolling bounce of your consumers and then verify that your consumers are healthy.
-    
      5. 
-    
    
-    A roll-back (i.e., migrating from Kafka back to ZooKeeper) can also be performed using the above steps if you set offsets.storage=zookeeper.
+    When the coordinator receives an offset fetch request, it simply returns the last committed offset vector from the offsets cache.
+        In case coordinator was just started or if it just became the coordinator for a new set of consumer groups (by becoming a leader for a partition of the offsets topic),
+        it may need to load the offsets topic partition into the cache. In this case, the offset fetch will fail with an
+        CoordinatorLoadInProgressException and the consumer may retry the OffsetFetchRequest after backing off.
     
    
 
     
    ZooKeeper Directories
    
@@ -287,47 +285,6 @@ 
    Broker Topic Registry
    
     Each broker registers itself under the topics it maintains and stores the number of partitions for that topic.
     
    
 
-    
    Consumers and Consumer Groups
    
-    
    
-    Consumers of topics also register themselves in ZooKeeper, in order to coordinate with each other and balance the consumption of data. Consumers can also store their offsets in ZooKeeper by setting offsets.storage=zookeeper. However, this offset storage mechanism will be deprecated in a future release. Therefore, it is recommended to migrate offsets storage to Kafka.
-    
    
-
-    
    
-    Multiple consumers can form a group and jointly consume a single topic. Each consumer in the same group is given a shared group_id.
-    For example if one consumer is your foobar process, which is run across three machines, then you might assign this group of consumers the id "foobar". This group id is provided in the configuration of the consumer, and is your way to tell the consumer which group it belongs to.
-    
    
-
-    
    
-    The consumers in a group divide up the partitions as fairly as possible, each partition is consumed by exactly one consumer in a consumer group.
-    
    
-
-    
    Consumer Id Registry
    
-    
    
-    In addition to the group_id which is shared by all consumers in a group, each consumer is given a transient, unique consumer_id (of the form hostname:uuid) for identification purposes. Consumer ids are registered in the following directory.
-    
    -    /consumers/[group_id]/ids/[consumer_id] --> {"version":...,"subscription":{...:...},"pattern":...,"timestamp":...} (ephemeral node)
-    
    
-    Each of the consumers in the group registers under its group and creates a znode with its consumer_id. The value of the znode contains a map of <topic, #streams>. This id is simply used to identify each of the consumers which is currently active within a group. This is an ephemeral node so it will disappear if the consumer process dies.
-    
    
-
-    
    Consumer Offsets
    
-    
    
-    Consumers track the maximum offset they have consumed in each partition. This value is stored in a ZooKeeper directory if offsets.storage=zookeeper.
-    
    
-    
    -    /consumers/[group_id]/offsets/[topic]/[partition_id] --> offset_counter_value (persistent node)
-    
    
-
-    
    Partition Owner registry
    
-
-    
    
-    Each broker partition is consumed by a single consumer within a given consumer group. The consumer must establish its ownership of a given partition before any consumption can begin. To establish its ownership, a consumer writes its own id in an ephemeral node under the particular broker partition it is claiming.
-    
    
-
-    
    -    /consumers/[group_id]/owners/[topic]/[partition_id] --> consumer_node_id (ephemeral node)
-    
    
-
     
    Cluster Id
    
 
     
    
@@ -342,45 +299,6 @@ 
    Broker node
     
    
     The broker nodes are basically independent, so they only publish information about what they have. When a broker joins, it registers itself under the broker node registry directory and writes information about its host name and port. The broker also register the list of existing topics and their logical partitions in the broker topic registry. New topics are registered dynamically when they are created on the broker.
     
    
-
-    
    Consumer registration algorithm
    
-
-    
    
-    When a consumer starts, it does the following:
-    
      
-    
    1.  Register itself in the consumer id registry under its group.
-    
      2. 
-    
    2.  Register a watch on changes (new consumers joining or any existing consumers leaving) under the consumer id registry. (Each change triggers rebalancing among all consumers within the group to which the changed consumer belongs.)
-    
      3. 
-    
    3.  Register a watch on changes (new brokers joining or any existing brokers leaving) under the broker id registry. (Each change triggers rebalancing among all consumers in all consumer groups.) 
      4. 
-    
    4.  If the consumer creates a message stream using a topic filter, it also registers a watch on changes (new topics being added) under the broker topic registry. (Each change will trigger re-evaluation of the available topics to determine which topics are allowed by the topic filter. A new allowed topic will trigger rebalancing among all consumers within the consumer group.)
      5. 
-    
    5.  Force itself to rebalance within in its consumer group.
-    
      6. 
-    
    
-    
    
-
-    
    Consumer rebalancing algorithm
    
-    
    
-    The consumer rebalancing algorithms allows all the consumers in a group to come into consensus on which consumer is consuming which partitions. Consumer rebalancing is triggered on each addition or removal of both broker nodes and other consumers within the same group. For a given topic and a given consumer group, broker partitions are divided evenly among consumers within the group. A partition is always consumed by a single consumer. This design simplifies the implementation. Had we allowed a partition to be concurrently consumed by multiple consumers, there would be contention on the partition and some kind of locking would be required. If there are more consumers than partitions, some consumers won't get any data at all. During rebalancing, we try to assign partitions to consumers in such a way that reduces the number of broker nodes each consumer has to connect to.
-    
    
-    
    
-    Each consumer does the following during rebalancing:
-    
    
-    
    -    1. For each topic T that Ci subscribes to
-    2.   let PT be all partitions producing topic T
-    3.   let CG be all consumers in the same group as Ci that consume topic T
-    4.   sort PT (so partitions on the same broker are clustered together)
-    5.   sort CG
-    6.   let i be the index position of Ci in CG and let N = size(PT)/size(CG)
-    7.   assign partitions from i*N to (i+1)*N - 1 to consumer Ci
-    8.   remove current entries owned by Ci from the partition owner registry
-    9.   add newly assigned partitions to the partition owner registry
-            (we may need to re-try this until the original partition owner releases its ownership)
-    
    
-    
    
-    When rebalancing is triggered at one consumer, rebalancing should be triggered in other consumers within the same group about the same time.
-    
    
 
 
 
    
diff --git a/docs/ops.html b/docs/ops.html
index 1445bfd2d253f..9d4f6cc4dfe6e 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -127,12 +127,7 @@ 
    Mirroring data
         --producer.config producer.properties --whitelist my-topic
   
   Note that we specify the list of topics with the --whitelist option. This option allows any regular expression using Java-style regular expressions. So you could mirror two topics named A and B using --whitelist 'A|B'. Or you could mirror all topics using --whitelist '*'. Make sure to quote any regular expression to ensure the shell doesn't try to expand it as a file path. For convenience we allow the use of ',' instead of '|' to specify a list of topics.
-  
    
-  Sometimes it is easier to say what it is that you don't want. Instead of using --whitelist to say what you want
-  to mirror you can use --blacklist to say what to exclude. This also takes a regular expression argument.
-  However, --blacklist is not supported when the new consumer has been enabled (i.e. when bootstrap.servers
-  has been defined in the consumer configuration).
-  
    
+
   Combining mirroring with the configuration auto.create.topics.enable=true makes it possible to have a replica cluster that will automatically create and replicate all data in a source cluster even as new topics are added.
 
   
    Checking consumer position
    
@@ -140,29 +135,15 @@ 
    Checking consu
   
       > bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-group
 
-  Note: This will only show information about consumers that use the Java consumer API (non-ZooKeeper-based consumers).
-
   TOPIC                          PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG        CONSUMER-ID                                       HOST                           CLIENT-ID
   my-topic                       0          2               4               2          consumer-1-029af89c-873c-4751-a720-cefd41a669d6   /127.0.0.1                     consumer-1
   my-topic                       1          2               3               1          consumer-1-029af89c-873c-4751-a720-cefd41a669d6   /127.0.0.1                     consumer-1
   my-topic                       2          2               3               1          consumer-2-42c1abd4-e3b2-425d-a8bb-e1ea49b29bb2   /127.0.0.1                     consumer-2
   
    
 
-  This tool also works with ZooKeeper-based consumers:
-  
    -  > bin/kafka-consumer-groups.sh --zookeeper localhost:2181 --describe --group my-group
-
-  Note: This will only show information about consumers that use ZooKeeper (not those using the Java consumer API).
-
-  TOPIC                          PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG        CONSUMER-ID
-  my-topic                       0          2               4               2          my-group_consumer-1
-  my-topic                       1          2               3               1          my-group_consumer-1
-  my-topic                       2          2               3               1          my-group_consumer-2
-  
    
-
   
    Managing Consumer Groups
    
 
-  With the ConsumerGroupCommand tool, we can list, describe, or delete consumer groups. When using the new consumer API (where the broker handles coordination of partition handling and rebalance), the group can be deleted manually, or automatically when the last committed offset for that group expires. Manual deletion works only if the group does not have any active members.
+  With the ConsumerGroupCommand tool, we can list, describe, or delete the consumer groups. The consumer group can be deleted manually, or automatically when the last committed offset for that group expires. Manual deletion works only if the group does not have any active members.
 
   For example, to list all consumer groups across all topics:
 
@@ -186,7 +167,7 @@ 
    Managing C
   topic3          2          243655          398812          155157          consumer4-117fe4d3-c6c1-4178-8ee9-eb4a3954bee0 /127.0.0.1      consumer4
   
 
-  There are a number of additional "describe" options that can be used to provide more detailed information about a consumer group that uses the new consumer API:
+  There are a number of additional "describe" options that can be used to provide more detailed information about a consumer group:
   
      
     
    • --members: This option provides the list of all active members in the consumer group.
       
      @@ -225,7 +206,6 @@ 
      Managing C
   
         > bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --delete --group my-group --group my-other-group
 
-  Note: This will not show information about old Zookeeper-based consumers.
   Deletion of requested consumer groups ('my-group', 'my-other-group') was successful.
   
      
 
@@ -660,14 +640,8 @@ 
      6.2 Datacenters
      
   
      6.3 Kafka Configuration
      
 
   
      Important Client Configurations
      
-  The most important old Scala producer configurations control
-  
        
-      
      • acks
        • 
-      
      • compression
        • 
-      
      • sync vs async production
        • 
-      
      • batch size (for async producers)
        • 
-  
      
-  The most important new Java producer configurations control
+
+  The most important producer configurations are:
   
        
       
      • acks
        • 
       
      • compression
        • 
@@ -805,7 +779,7 @@ 
        EXT4 Notes
        
 
   
        6.6 Monitoring
        
 
-  Kafka uses Yammer Metrics for metrics reporting in the server and Scala clients. The Java clients use Kafka Metrics, a built-in metrics registry that minimizes transitive dependencies pulled into client applications. Both expose metrics via JMX and can be configured to report stats using pluggable stats reporters to hook up to your monitoring system.
+  Kafka uses Yammer Metrics for metrics reporting in the server. The Java clients use Kafka Metrics, a built-in metrics registry that minimizes transitive dependencies pulled into client applications. Both expose metrics via JMX and can be configured to report stats using pluggable stats reporters to hook up to your monitoring system.
   
        
   All Kafka rate metrics have a corresponding cumulative count metric with suffix -total. For example,
   records-consumed-rate has a corresponding metric named records-consumed-total.
@@ -985,10 +959,7 @@ 
        6.6 Monitoring
        
       
       
         Number of messages the consumer lags behind the producer by. Published by the consumer, not broker.
-        
-          
        Old consumer: kafka.consumer:type=ConsumerFetcherManager,name=MaxLag,clientId=([-.\w]+)
        
-          
        New consumer: kafka.consumer:type=consumer-fetch-manager-metrics,client-id={client-id} Attribute: records-lag-max
        
-        
+        kafka.consumer:type=consumer-fetch-manager-metrics,client-id={client-id} Attribute: records-lag-max
         
       
       
@@ -1295,11 +1266,11 @@ 
        Produc
   
 
 
-  
        New consumer monitoring
        
+  
        consumer monitoring
        
 
-  The following metrics are available on new consumer instances.
+  The following metrics are available on consumer instances.
 
-  
        Consumer Group Metrics
        
+  
        Consumer Group Metrics
        
   
@@ -1395,7 +1366,7 @@ 
        
     
        
     
       
        
   
        
 
-  
        Consumer Fetch Metrics
        
+  
        Consumer Fetch Metrics
        
 
   
 
diff --git a/docs/security.html b/docs/security.html
index 743d673fe8048..d7859e08d72c5 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -1286,7 +1286,7 @@ 
        7.6.1 New clusters
        
         
      •  Set the configuration property zookeeper.set.acl in each broker to true
        • 
     
 
-    The metadata stored in ZooKeeper for the Kafka cluster is world-readable, but can only be modified by the brokers. The rationale behind this decision is that the data stored in ZooKeeper is not sensitive, but inappropriate manipulation of that data can cause cluster disruption. We also recommend limiting the access to ZooKeeper via network segmentation (only brokers and some admin tools need access to ZooKeeper if the new Java consumer and producer clients are used).
+    The metadata stored in ZooKeeper for the Kafka cluster is world-readable, but can only be modified by the brokers. The rationale behind this decision is that the data stored in ZooKeeper is not sensitive, but inappropriate manipulation of that data can cause cluster disruption. We also recommend limiting the access to ZooKeeper via network segmentation (only brokers and some admin tools need access to ZooKeeper).
 
     
        7.6.2 Migrating clusters
        
     If you are running a version of Kafka that does not support security or simply with security disabled, and you want to make the cluster secure, then you need to execute the following steps to enable ZooKeeper authentication with minimal disruption to your operations:
diff --git a/docs/toc.html b/docs/toc.html
index e7d939e72b776..f1897fe3a0cff 100644
--- a/docs/toc.html
+++ b/docs/toc.html
@@ -36,7 +36,6 @@
                 
      • 2.3 Streams API
                 
      • 2.4 Connect API
                 
      • 2.5 AdminClient API
-                
      • 2.6 Legacy APIs
             
      
         
      • 
         
    • 3. Configuration
@@ -45,10 +44,6 @@
                 
    • 3.2 Topic Configs
                 
    • 3.3 Producer Configs
                 
    • 3.4 Consumer Configs
-                    
                 
    • 3.5 Kafka Connect Configs
                 
    • 3.6 Kafka Streams Configs
                 
    • 3.7 AdminClient Configs

From a289865266618d9736fe49d11edfbc2b146f5148 Mon Sep 17 00:00:00 2001
From: Joan Goyeau 
Date: Tue, 21 Aug 2018 22:41:59 +0100
Subject: [PATCH 0733/1847] MINOR: Small refactorings on KTable joins (#5540)

Reviewers: Guozhang Wang 
---
 .../apache/kafka/streams/kstream/internals/KTableImpl.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index bbec96c710065..352e42d391889 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -359,8 +359,6 @@ public  KTable join(final KTable other,
     public  KTable join(final KTable other,
                                        final ValueJoiner joiner,
                                        final Materialized> materialized) {
-        Objects.requireNonNull(other, "other can't be null");
-        Objects.requireNonNull(joiner, "joiner can't be null");
         Objects.requireNonNull(materialized, "materialized can't be null");
         final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
         materializedInternal.generateStoreNameIfNeeded(builder, MERGE_NAME);
@@ -378,6 +376,7 @@ public  KTable outerJoin(final KTable other,
     public  KTable outerJoin(final KTable other,
                                             final ValueJoiner joiner,
                                             final Materialized> materialized) {
+        Objects.requireNonNull(materialized, "materialized can't be null");
         final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
         materializedInternal.generateStoreNameIfNeeded(builder, MERGE_NAME);
 
@@ -394,8 +393,10 @@ public  KTable leftJoin(final KTable other,
     public  KTable leftJoin(final KTable other,
                                            final ValueJoiner joiner,
                                            final Materialized> materialized) {
+        Objects.requireNonNull(materialized, "materialized can't be null");
         final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
         materializedInternal.generateStoreNameIfNeeded(builder, MERGE_NAME);
+
         return doJoin(other, joiner, materializedInternal, true, false);
     }
 
@@ -410,7 +411,6 @@ private  KTable doJoin(final KTable other,
         final String internalQueryableName = materializedInternal == null ? null : materializedInternal.storeName();
         final String joinMergeName = builder.newProcessorName(MERGE_NAME);
 
-
         return buildJoin(
             (AbstractStream) other,
             joiner,

From dae9c41838e745d51dba4e6be18359d42d5ff8a3 Mon Sep 17 00:00:00 2001
From: Joan Goyeau 
Date: Tue, 21 Aug 2018 23:41:36 +0100
Subject: [PATCH 0734/1847] KAFKA-7301: Fix streams Scala join ambiguous
 overload (#5502)

Join in the Scala streams API is currently unusable in 2.0.0 as reported by @mowczare:
#5019 (comment)

This due to an overload of it with the same signature in the first curried parameter.
See compiler issue that didn't catch it: https://issues.scala-lang.org/browse/SI-2628

Reviewers: Debasish Ghosh , Guozhang Wang , John Roesler 
---
 build.gradle                                  |  1 +
 .../kafka/streams/scala/kstream/KTable.scala  | 16 ++--
 .../kafka/streams/scala/KStreamTest.scala     | 70 ++++++++++++++++
 .../kafka/streams/scala/KTableTest.scala      | 79 +++++++++++++++++++
 ...inScalaIntegrationTestImplicitSerdes.scala | 16 +---
 ...mToTableJoinScalaIntegrationTestBase.scala |  2 +-
 .../StreamToTableJoinTestData.scala           |  2 +-
 .../streams/scala/utils/TestDriver.scala      | 52 ++++++++++++
 8 files changed, 213 insertions(+), 25 deletions(-)
 create mode 100644 streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
 create mode 100644 streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala
 rename streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/{ => utils}/StreamToTableJoinScalaIntegrationTestBase.scala (99%)
 rename streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/{ => utils}/StreamToTableJoinTestData.scala (97%)
 create mode 100644 streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala

diff --git a/build.gradle b/build.gradle
index c1387d4ef60bb..c963253a8bca9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1021,6 +1021,7 @@ project(':streams:streams-scala') {
     testCompile project(':core').sourceSets.test.output
     testCompile project(':streams').sourceSets.test.output
     testCompile project(':clients').sourceSets.test.output
+    testCompile project(':streams:test-utils')
 
     testCompile libs.junit
     testCompile libs.scalatest
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
index b66977193e11d..a78d321c941ea 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
@@ -20,6 +20,7 @@
 package org.apache.kafka.streams.scala
 package kstream
 
+import org.apache.kafka.common.serialization.Serde
 import org.apache.kafka.common.utils.Bytes
 import org.apache.kafka.streams.kstream.{KTable => KTableJ, _}
 import org.apache.kafka.streams.scala.ImplicitConversions._
@@ -245,9 +246,8 @@ class KTable[K, V](val inner: KTableJ[K, V]) {
    * one for each matched record-pair with the same key
    * @see `org.apache.kafka.streams.kstream.KTable#join`
    */
-  def join[VO, VR](other: KTable[K, VO])(
-    joiner: (V, VO) => VR,
-    materialized: Materialized[K, VR, ByteArrayKeyValueStore]
+  def join[VO, VR](other: KTable[K, VO], materialized: Materialized[K, VR, ByteArrayKeyValueStore])(
+    joiner: (V, VO) => VR
   ): KTable[K, VR] =
     inner.join[VO, VR](other.inner, joiner.asValueJoiner, materialized)
 
@@ -274,9 +274,8 @@ class KTable[K, V](val inner: KTableJ[K, V]) {
    * one for each matched record-pair with the same key
    * @see `org.apache.kafka.streams.kstream.KTable#leftJoin`
    */
-  def leftJoin[VO, VR](other: KTable[K, VO])(
-    joiner: (V, VO) => VR,
-    materialized: Materialized[K, VR, ByteArrayKeyValueStore]
+  def leftJoin[VO, VR](other: KTable[K, VO], materialized: Materialized[K, VR, ByteArrayKeyValueStore])(
+    joiner: (V, VO) => VR
   ): KTable[K, VR] =
     inner.leftJoin[VO, VR](other.inner, joiner.asValueJoiner, materialized)
 
@@ -303,9 +302,8 @@ class KTable[K, V](val inner: KTableJ[K, V]) {
    * one for each matched record-pair with the same key
    * @see `org.apache.kafka.streams.kstream.KTable#leftJoin`
    */
-  def outerJoin[VO, VR](other: KTable[K, VO])(
-    joiner: (V, VO) => VR,
-    materialized: Materialized[K, VR, ByteArrayKeyValueStore]
+  def outerJoin[VO, VR](other: KTable[K, VO], materialized: Materialized[K, VR, ByteArrayKeyValueStore])(
+    joiner: (V, VO) => VR
   ): KTable[K, VR] =
     inner.outerJoin[VO, VR](other.inner, joiner.asValueJoiner, materialized)
 
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
new file mode 100644
index 0000000000000..6a302b207a9cc
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2018 Joan Goyeau.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import org.apache.kafka.streams.kstream.JoinWindows
+import org.apache.kafka.streams.scala.Serdes._
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.utils.TestDriver
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{FlatSpec, Matchers}
+
+@RunWith(classOf[JUnitRunner])
+class KStreamTest extends FlatSpec with Matchers with TestDriver {
+
+  "selectKey a KStream" should "select a new key" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+    val sinkTopic = "sink"
+
+    builder.stream[String, String](sourceTopic).selectKey((_, value) => value).to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    testDriver.readRecord[String, String](sinkTopic).key shouldBe "value1"
+
+    testDriver.pipeRecord(sourceTopic, ("1", "value2"))
+    testDriver.readRecord[String, String](sinkTopic).key shouldBe "value2"
+
+    testDriver.close()
+  }
+
+  "join 2 KStreams" should "join correctly records" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic1 = "source1"
+    val sourceTopic2 = "source2"
+    val sinkTopic = "sink"
+
+    val stream1 = builder.stream[String, String](sourceTopic1)
+    val stream2 = builder.stream[String, String](sourceTopic2)
+    stream1.join(stream2)((a, b) => s"$a-$b", JoinWindows.of(1000)).to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic1, ("1", "topic1value1"))
+    testDriver.pipeRecord(sourceTopic2, ("1", "topic2value1"))
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "topic1value1-topic2value1"
+
+    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+}
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala
new file mode 100644
index 0000000000000..8c88ff5066f0e
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2018 Joan Goyeau.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala
+
+import org.apache.kafka.streams.kstream.Materialized
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala.Serdes._
+import org.apache.kafka.streams.scala.utils.TestDriver
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{FlatSpec, Matchers}
+
+@RunWith(classOf[JUnitRunner])
+class KTableTest extends FlatSpec with Matchers with TestDriver {
+
+  "join 2 KTables" should "join correctly records" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic1 = "source1"
+    val sourceTopic2 = "source2"
+    val sinkTopic = "sink"
+
+    val table1 = builder.stream[String, String](sourceTopic1).groupBy((key, _) => key).count()
+    val table2 = builder.stream[String, String](sourceTopic2).groupBy((key, _) => key).count()
+    table1.join(table2)((a, b) => a + b).toStream.to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic1, ("1", "topic1value1"))
+    testDriver.pipeRecord(sourceTopic2, ("1", "topic2value1"))
+    testDriver.readRecord[String, Long](sinkTopic).value shouldBe 2
+
+    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+
+  "join 2 KTables with a Materialized" should "join correctly records and state store" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic1 = "source1"
+    val sourceTopic2 = "source2"
+    val sinkTopic = "sink"
+    val stateStore = "store"
+    val materialized = Materialized
+      .as[String, Long, ByteArrayKeyValueStore](stateStore)
+      .withKeySerde(Serdes.String)
+      .withValueSerde(Serdes.Long)
+
+    val table1 = builder.stream[String, String](sourceTopic1).groupBy((key, _) => key).count()
+    val table2 = builder.stream[String, String](sourceTopic2).groupBy((key, _) => key).count()
+    table1.join(table2, materialized)((a, b) => a + b).toStream.to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic1, ("1", "topic1value1"))
+    testDriver.pipeRecord(sourceTopic2, ("1", "topic2value1"))
+    testDriver.readRecord[String, Long](sinkTopic).value shouldBe 2
+    testDriver.getKeyValueStore[String, Long](stateStore).get("1") shouldBe 2
+
+    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+}
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala
index 3d1bab5d086ca..fd5f361a5b60a 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestImplicitSerdes.scala
@@ -18,20 +18,11 @@ package org.apache.kafka.streams.scala
 
 import java.util.Properties
 
-import org.apache.kafka.clients.consumer.ConsumerConfig
-import org.apache.kafka.clients.producer.ProducerConfig
-import org.apache.kafka.common.serialization._
-import org.apache.kafka.common.utils.MockTime
 import org.apache.kafka.streams._
-import org.apache.kafka.streams.integration.utils.{EmbeddedKafkaCluster, IntegrationTestUtils}
-import org.apache.kafka.streams.processor.internals.StreamThread
 import org.apache.kafka.streams.scala.ImplicitConversions._
 import org.apache.kafka.streams.scala.kstream._
-import org.apache.kafka.test.TestUtils
-import org.junit.Assert._
+import org.apache.kafka.streams.scala.utils.StreamToTableJoinScalaIntegrationTestBase
 import org.junit._
-import org.junit.rules.TemporaryFolder
-import org.scalatest.junit.JUnitSuite
 
 /**
  * Test suite that does an example to demonstrate stream-table joins in Kafka Streams
@@ -141,10 +132,7 @@ class StreamToTableJoinScalaIntegrationTestImplicitSerdes extends StreamToTableJ
     val streams: KafkaStreamsJ = new KafkaStreamsJ(builder.build(), streamsConfiguration)
 
     streams.start()
-
-    val actualClicksPerRegion: java.util.List[KeyValue[String, Long]] =
-      produceNConsume(userClicksTopicJ, userRegionsTopicJ, outputTopicJ)
-
+    produceNConsume(userClicksTopicJ, userRegionsTopicJ, outputTopicJ)
     streams.close()
   }
 }
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestBase.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/StreamToTableJoinScalaIntegrationTestBase.scala
similarity index 99%
rename from streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestBase.scala
rename to streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/StreamToTableJoinScalaIntegrationTestBase.scala
index cf87eb5e27d5c..9a3ee7f27942e 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinScalaIntegrationTestBase.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/StreamToTableJoinScalaIntegrationTestBase.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.streams.scala
+package org.apache.kafka.streams.scala.utils
 
 import java.util.Properties
 
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinTestData.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/StreamToTableJoinTestData.scala
similarity index 97%
rename from streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinTestData.scala
rename to streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/StreamToTableJoinTestData.scala
index e9040eee5d456..890d8c2ee14b2 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/StreamToTableJoinTestData.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/StreamToTableJoinTestData.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.kafka.streams.scala
+package org.apache.kafka.streams.scala.utils
 
 import org.apache.kafka.streams.KeyValue
 
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala
new file mode 100644
index 0000000000000..1497dd747931d
--- /dev/null
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/utils/TestDriver.scala
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2018 Joan Goyeau.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.scala.utils
+
+import java.util.{Properties, UUID}
+
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.kafka.common.serialization.Serde
+import org.apache.kafka.streams.scala.StreamsBuilder
+import org.apache.kafka.streams.test.ConsumerRecordFactory
+import org.apache.kafka.streams.{StreamsConfig, TopologyTestDriver}
+import org.scalatest.Suite
+
+trait TestDriver { this: Suite =>
+
+  def createTestDriver(builder: StreamsBuilder, initialWallClockTimeMs: Long = System.currentTimeMillis()) = {
+    val config = new Properties()
+    config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test")
+    config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234")
+    config.put(StreamsConfig.STATE_DIR_CONFIG, s"out/state-store-${UUID.randomUUID()}")
+    new TopologyTestDriver(builder.build(), config, initialWallClockTimeMs)
+  }
+
+  implicit class TopologyTestDriverOps(inner: TopologyTestDriver) {
+    def pipeRecord[K, V](topic: String, record: (K, V), timestampMs: Long = System.currentTimeMillis())(
+      implicit serdeKey: Serde[K],
+      serdeValue: Serde[V]
+    ): Unit = {
+      val recordFactory = new ConsumerRecordFactory[K, V](serdeKey.serializer, serdeValue.serializer)
+      inner.pipeInput(recordFactory.create(topic, record._1, record._2, timestampMs))
+    }
+
+    def readRecord[K, V](topic: String)(implicit serdeKey: Serde[K], serdeValue: Serde[V]): ProducerRecord[K, V] =
+      inner.readOutput(topic, serdeKey.deserializer, serdeValue.deserializer)
+  }
+}

From 3511e904effbfb74fa54c26467477863b7f730c0 Mon Sep 17 00:00:00 2001
From: Robert Yokota 
Date: Tue, 21 Aug 2018 23:42:56 -0700
Subject: [PATCH 0735/1847] MINOR: Split at first occurrence of '=' in kafka.py
 props parsing (#5549)

This is a fix to #5226 to account for config properties that have an
equal char in the value.   Otherwise if there is one
equal char in the value the following error occurs:

dictionary update sequence element #XX has length 3; 2 is required

Reviewers: Colin Patrick McCabe , Ismael Juma 
---
 tests/kafkatest/services/kafka/kafka.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index b0a9faacf2eca..c3cc9f762bff8 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -217,7 +217,7 @@ def prop_file(self, node):
         config_template = self.render('kafka.properties', node=node, broker_id=self.idx(node),
                                  security_config=self.security_config, num_nodes=self.num_nodes)
 
-        configs = dict( l.rstrip().split('=') for l in config_template.split('\n')
+        configs = dict( l.rstrip().split('=', 1) for l in config_template.split('\n')
                         if not l.startswith("#") and "=" in l )
 
         #load specific test override configs

From bf0675ac5f31d67a1c75baf211475b8fc98f1ef1 Mon Sep 17 00:00:00 2001
From: Alex Dunayevsky 
Date: Wed, 22 Aug 2018 20:13:10 +0300
Subject: [PATCH 0736/1847] =?UTF-8?q?KAFKA-6343=20Documentation=20update?=
 =?UTF-8?q?=20in=20OS-level=20tuning=20section:=20add=20vm.ma=E2=80=A6=20(?=
 =?UTF-8?q?#4358)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reviewers: Manikumar Reddy O , Ismael Juma , Jun Rao 
---
 docs/ops.html | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/ops.html b/docs/ops.html
index 9d4f6cc4dfe6e..d57f1cf20ffad 100644
--- a/docs/ops.html
+++ b/docs/ops.html
@@ -705,10 +705,11 @@ 
      OS
      
   
      
   We have seen a few issues running on Windows and Windows is not currently a well supported platform though we would be happy to change that.
   
      
-  It is unlikely to require much OS-level tuning, but there are two potentially important OS-level configurations:
+  It is unlikely to require much OS-level tuning, but there are three potentially important OS-level configurations:
   
        
-      
      • File descriptor limits: Kafka uses file descriptors for log segments and open connections.  If a broker hosts many partitions, consider that the broker needs at least (number_of_partitions)*(partition_size/segment_size) to track all log segments in addition to the number of connections the broker makes.  We recommend at least 100000 allowed file descriptors for the broker processes as a starting point.
+      
      • File descriptor limits: Kafka uses file descriptors for log segments and open connections.  If a broker hosts many partitions, consider that the broker needs at least (number_of_partitions)*(partition_size/segment_size) to track all log segments in addition to the number of connections the broker makes. We recommend at least 100000 allowed file descriptors for the broker processes as a starting point. Note: The mmap() function adds an extra reference to the file associated with the file descriptor fildes which is not removed by a subsequent close() on that file descriptor. This reference is removed when there are no more mappings to the file.
       
      • Max socket buffer size: can be increased to enable high-performance data transfer between data centers as described here.
+      
      • Maximum number of memory map areas a process may have (aka vm.max_map_count). See the Linux kernel documentation. You should keep an eye at this OS-level property when considering the maximum number of partitions a broker may have. By default, on a number of Linux systems, the value of vm.max_map_count is somewhere around 65535. Each log segment, allocated per partition, requires a pair of index/timeindex files, and each of these files consumes 1 map area. In other words, each log segment uses 2 map areas. Thus, each partition requires minimum 2 map areas, as long as it hosts a single log segment. That is to say, creating 50000 partitions on a broker will result allocation of 100000 map areas and likely cause broker crash with OutOfMemoryError (Map failed) on a system with default vm.max_map_count. Keep in mind that the number of log segments per partition varies depending on the segment size, load intensity, retention policy and, generally, tends to be more than one.
   
      
   
      
 

From 8f562c1a8d16b940f898a57ec508318c397f6f13 Mon Sep 17 00:00:00 2001
From: Viktor Somogyi 
Date: Thu, 23 Aug 2018 10:36:18 +0200
Subject: [PATCH 0737/1847] MINOR: Fix side-effecting nullary methods warning
 in JsonValueTest (#5493)

Reviewers: Ismael Juma 
---
 .../unit/kafka/utils/json/JsonValueTest.scala | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/utils/json/JsonValueTest.scala b/core/src/test/scala/unit/kafka/utils/json/JsonValueTest.scala
index b12d0f396b84e..640feedff46b7 100644
--- a/core/src/test/scala/unit/kafka/utils/json/JsonValueTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/json/JsonValueTest.scala
@@ -25,7 +25,7 @@ import kafka.utils.Json
 
 class JsonValueTest {
 
-  val json = """
+  private val json = """
     |{
     |  "boolean": false,
     |  "int": 1234,
@@ -66,7 +66,7 @@ class JsonValueTest {
   }
 
   @Test
-  def testAsJsonObject: Unit = {
+  def testAsJsonObject(): Unit = {
     val parsed = parse(json).asJsonObject
     val obj = parsed("object")
     assertEquals(obj, obj.asJsonObject)
@@ -74,14 +74,14 @@ class JsonValueTest {
   }
 
   @Test
-  def testAsJsonObjectOption: Unit = {
+  def testAsJsonObjectOption(): Unit = {
     val parsed = parse(json).asJsonObject
     assertTrue(parsed("object").asJsonObjectOption.isDefined)
     assertEquals(None, parsed("array").asJsonObjectOption)
   }
 
   @Test
-  def testAsJsonArray: Unit = {
+  def testAsJsonArray(): Unit = {
     val parsed = parse(json).asJsonObject
     val array = parsed("array")
     assertEquals(array, array.asJsonArray)
@@ -89,28 +89,28 @@ class JsonValueTest {
   }
 
   @Test
-  def testAsJsonArrayOption: Unit = {
+  def testAsJsonArrayOption(): Unit = {
     val parsed = parse(json).asJsonObject
     assertTrue(parsed("array").asJsonArrayOption.isDefined)
     assertEquals(None, parsed("object").asJsonArrayOption)
   }
 
   @Test
-  def testJsonObjectGet: Unit = {
+  def testJsonObjectGet(): Unit = {
     val parsed = parse(json).asJsonObject
     assertEquals(Some(parse("""{"a":true,"b":false}""")), parsed.get("object"))
     assertEquals(None, parsed.get("aaaaa"))
   }
 
   @Test
-  def testJsonObjectApply: Unit = {
+  def testJsonObjectApply(): Unit = {
     val parsed = parse(json).asJsonObject
     assertEquals(parse("""{"a":true,"b":false}"""), parsed("object"))
     assertThrow[JsonMappingException](parsed("aaaaaaaa"))
   }
 
   @Test
-  def testJsonObjectIterator: Unit = {
+  def testJsonObjectIterator(): Unit = {
     assertEquals(
       Vector("a" -> parse("true"), "b" -> parse("false")),
       parse(json).asJsonObject("object").asJsonObject.iterator.toVector
@@ -118,12 +118,12 @@ class JsonValueTest {
   }
 
   @Test
-  def testJsonArrayIterator: Unit = {
+  def testJsonArrayIterator(): Unit = {
     assertEquals(Vector("4.0", "11.1", "44.5").map(parse), parse(json).asJsonObject("array").asJsonArray.iterator.toVector)
   }
 
   @Test
-  def testJsonValueEquals: Unit = {
+  def testJsonValueEquals(): Unit = {
 
     assertEquals(parse(json), parse(json))
 
@@ -139,24 +139,24 @@ class JsonValueTest {
   }
 
   @Test
-  def testJsonValueHashCode: Unit = {
+  def testJsonValueHashCode(): Unit = {
     assertEquals(new ObjectMapper().readTree(json).hashCode, parse(json).hashCode)
   }
 
   @Test
-  def testJsonValueToString: Unit = {
+  def testJsonValueToString(): Unit = {
     val js = """{"boolean":false,"int":1234,"array":[4.0,11.1,44.5],"object":{"a":true,"b":false}}"""
     assertEquals(js, parse(js).toString)
   }
 
   @Test
-  def testDecodeBoolean: Unit = {
+  def testDecodeBoolean(): Unit = {
     assertTo[Boolean](false, _("boolean"))
     assertToFails[Boolean](_("int"))
   }
 
   @Test
-  def testDecodeString: Unit = {
+  def testDecodeString(): Unit = {
     assertTo[String]("string", _("string"))
     assertTo[String]("123", _("number_as_string"))
     assertToFails[String](_("int"))
@@ -164,20 +164,20 @@ class JsonValueTest {
   }
 
   @Test
-  def testDecodeInt: Unit = {
+  def testDecodeInt(): Unit = {
     assertTo[Int](1234, _("int"))
     assertToFails[Int](_("long"))
   }
 
   @Test
-  def testDecodeLong: Unit = {
+  def testDecodeLong(): Unit = {
     assertTo[Long](3000000000L, _("long"))
     assertTo[Long](1234, _("int"))
     assertToFails[Long](_("string"))
   }
 
   @Test
-  def testDecodeDouble: Unit = {
+  def testDecodeDouble(): Unit = {
     assertTo[Double](16.244355, _("double"))
     assertTo[Double](1234.0, _("int"))
     assertTo[Double](3000000000L, _("long"))
@@ -185,7 +185,7 @@ class JsonValueTest {
   }
 
   @Test
-  def testDecodeSeq: Unit = {
+  def testDecodeSeq(): Unit = {
     assertTo[Seq[Double]](Seq(4.0, 11.1, 44.5), _("array"))
     assertToFails[Seq[Double]](_("string"))
     assertToFails[Seq[Double]](_("object"))
@@ -193,7 +193,7 @@ class JsonValueTest {
   }
 
   @Test
-  def testDecodeMap: Unit = {
+  def testDecodeMap(): Unit = {
     assertTo[Map[String, Boolean]](Map("a" -> true, "b" -> false), _("object"))
     assertToFails[Map[String, Int]](_("object"))
     assertToFails[Map[String, String]](_("object"))
@@ -201,7 +201,7 @@ class JsonValueTest {
   }
 
   @Test
-  def testDecodeOption: Unit = {
+  def testDecodeOption(): Unit = {
     assertTo[Option[Int]](None, _("null"))
     assertTo[Option[Int]](Some(1234), _("int"))
     assertToFails[Option[String]](_("int"))

From 4b2af9de6b3330bc7d9098ff690d80ef64628a6c Mon Sep 17 00:00:00 2001
From: Viktor Somogyi 
Date: Thu, 23 Aug 2018 10:44:04 +0200
Subject: [PATCH 0738/1847] MINOR: Use method references instead of anonymus
 classes in Errors (#5525)

Remove `ApiExceptionBuilder` in favour of `Function`.

Reviewers: Ismael Juma 
---
 .../apache/kafka/common/protocol/Errors.java  | 551 +++---------------
 1 file changed, 95 insertions(+), 456 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index 9c522dff935ec..090dca32651bb 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -96,6 +96,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * This class contains all the client-server errors--those errors that must be sent from the server to the client. These
@@ -110,533 +111,171 @@
  */
 public enum Errors {
     UNKNOWN_SERVER_ERROR(-1, "The server experienced an unexpected error when processing the request",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnknownServerException(message);
-            }
-        }),
-    NONE(0, null,
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return null;
-            }
-        }),
+            UnknownServerException::new),
+    NONE(0, null, message -> null),
     OFFSET_OUT_OF_RANGE(1, "The requested offset is not within the range of offsets maintained by the server.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new OffsetOutOfRangeException(message);
-            }
-        }),
+            OffsetOutOfRangeException::new),
     CORRUPT_MESSAGE(2, "This message has failed its CRC checksum, exceeds the valid size, or is otherwise corrupt.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new CorruptRecordException(message);
-            }
-        }),
+            CorruptRecordException::new),
     UNKNOWN_TOPIC_OR_PARTITION(3, "This server does not host this topic-partition.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnknownTopicOrPartitionException(message);
-            }
-        }),
+            UnknownTopicOrPartitionException::new),
     INVALID_FETCH_SIZE(4, "The requested fetch size is invalid.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidFetchSizeException(message);
-            }
-        }),
+            InvalidFetchSizeException::new),
     LEADER_NOT_AVAILABLE(5, "There is no leader for this topic-partition as we are in the middle of a leadership election.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new LeaderNotAvailableException(message);
-            }
-        }),
+            LeaderNotAvailableException::new),
     NOT_LEADER_FOR_PARTITION(6, "This server is not the leader for that topic-partition.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new NotLeaderForPartitionException(message);
-            }
-        }),
+            NotLeaderForPartitionException::new),
     REQUEST_TIMED_OUT(7, "The request timed out.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new TimeoutException(message);
-            }
-        }),
+            TimeoutException::new),
     BROKER_NOT_AVAILABLE(8, "The broker is not available.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new BrokerNotAvailableException(message);
-            }
-        }),
+            BrokerNotAvailableException::new),
     REPLICA_NOT_AVAILABLE(9, "The replica is not available for the requested topic-partition",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ReplicaNotAvailableException(message);
-            }
-        }),
+            ReplicaNotAvailableException::new),
     MESSAGE_TOO_LARGE(10, "The request included a message larger than the max message size the server will accept.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new RecordTooLargeException(message);
-            }
-        }),
+            RecordTooLargeException::new),
     STALE_CONTROLLER_EPOCH(11, "The controller moved to another broker.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ControllerMovedException(message);
-            }
-        }),
+            ControllerMovedException::new),
     OFFSET_METADATA_TOO_LARGE(12, "The metadata field of the offset request was too large.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new OffsetMetadataTooLarge(message);
-            }
-        }),
+            OffsetMetadataTooLarge::new),
     NETWORK_EXCEPTION(13, "The server disconnected before a response was received.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new NetworkException(message);
-            }
-        }),
+            NetworkException::new),
     COORDINATOR_LOAD_IN_PROGRESS(14, "The coordinator is loading and hence can't process requests.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new CoordinatorLoadInProgressException(message);
-            }
-        }),
+            CoordinatorLoadInProgressException::new),
     COORDINATOR_NOT_AVAILABLE(15, "The coordinator is not available.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new CoordinatorNotAvailableException(message);
-            }
-        }),
+            CoordinatorNotAvailableException::new),
     NOT_COORDINATOR(16, "This is not the correct coordinator.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new NotCoordinatorException(message);
-            }
-        }),
+            NotCoordinatorException::new),
     INVALID_TOPIC_EXCEPTION(17, "The request attempted to perform an operation on an invalid topic.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidTopicException(message);
-            }
-        }),
+            InvalidTopicException::new),
     RECORD_LIST_TOO_LARGE(18, "The request included message batch larger than the configured segment size on the server.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new RecordBatchTooLargeException(message);
-            }
-        }),
+            RecordBatchTooLargeException::new),
     NOT_ENOUGH_REPLICAS(19, "Messages are rejected since there are fewer in-sync replicas than required.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new NotEnoughReplicasException(message);
-            }
-        }),
+            NotEnoughReplicasException::new),
     NOT_ENOUGH_REPLICAS_AFTER_APPEND(20, "Messages are written to the log, but to fewer in-sync replicas than required.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new NotEnoughReplicasAfterAppendException(message);
-            }
-        }),
+            NotEnoughReplicasAfterAppendException::new),
     INVALID_REQUIRED_ACKS(21, "Produce request specified an invalid value for required acks.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidRequiredAcksException(message);
-            }
-        }),
+            InvalidRequiredAcksException::new),
     ILLEGAL_GENERATION(22, "Specified group generation id is not valid.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new IllegalGenerationException(message);
-            }
-        }),
+            IllegalGenerationException::new),
     INCONSISTENT_GROUP_PROTOCOL(23,
             "The group member's supported protocols are incompatible with those of existing members" +
                 " or first group member tried to join with empty protocol type or empty protocol list.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InconsistentGroupProtocolException(message);
-            }
-        }),
+            InconsistentGroupProtocolException::new),
     INVALID_GROUP_ID(24, "The configured groupId is invalid",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidGroupIdException(message);
-            }
-        }),
+            InvalidGroupIdException::new),
     UNKNOWN_MEMBER_ID(25, "The coordinator is not aware of this member.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnknownMemberIdException(message);
-            }
-        }),
+            UnknownMemberIdException::new),
     INVALID_SESSION_TIMEOUT(26,
             "The session timeout is not within the range allowed by the broker " +
             "(as configured by group.min.session.timeout.ms and group.max.session.timeout.ms).",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidSessionTimeoutException(message);
-            }
-        }),
+            InvalidSessionTimeoutException::new),
     REBALANCE_IN_PROGRESS(27, "The group is rebalancing, so a rejoin is needed.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new RebalanceInProgressException(message);
-            }
-        }),
+            RebalanceInProgressException::new),
     INVALID_COMMIT_OFFSET_SIZE(28, "The committing offset data size is not valid",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidCommitOffsetSizeException(message);
-            }
-        }),
+            InvalidCommitOffsetSizeException::new),
     TOPIC_AUTHORIZATION_FAILED(29, "Topic authorization failed.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new TopicAuthorizationException(message);
-            }
-        }),
+            TopicAuthorizationException::new),
     GROUP_AUTHORIZATION_FAILED(30, "Group authorization failed.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new GroupAuthorizationException(message);
-            }
-        }),
+            GroupAuthorizationException::new),
     CLUSTER_AUTHORIZATION_FAILED(31, "Cluster authorization failed.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ClusterAuthorizationException(message);
-            }
-        }),
+            ClusterAuthorizationException::new),
     INVALID_TIMESTAMP(32, "The timestamp of the message is out of acceptable range.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidTimestampException(message);
-            }
-        }),
+            InvalidTimestampException::new),
     UNSUPPORTED_SASL_MECHANISM(33, "The broker does not support the requested SASL mechanism.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnsupportedSaslMechanismException(message);
-            }
-        }),
+            UnsupportedSaslMechanismException::new),
     ILLEGAL_SASL_STATE(34, "Request is not valid given the current SASL state.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new IllegalSaslStateException(message);
-            }
-        }),
+            IllegalSaslStateException::new),
     UNSUPPORTED_VERSION(35, "The version of API is not supported.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnsupportedVersionException(message);
-            }
-        }),
+            UnsupportedVersionException::new),
     TOPIC_ALREADY_EXISTS(36, "Topic with this name already exists.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new TopicExistsException(message);
-            }
-        }),
+            TopicExistsException::new),
     INVALID_PARTITIONS(37, "Number of partitions is below 1.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidPartitionsException(message);
-            }
-        }),
+            InvalidPartitionsException::new),
     INVALID_REPLICATION_FACTOR(38, "Replication factor is below 1 or larger than the number of available brokers.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidReplicationFactorException(message);
-            }
-        }),
+            InvalidReplicationFactorException::new),
     INVALID_REPLICA_ASSIGNMENT(39, "Replica assignment is invalid.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidReplicaAssignmentException(message);
-            }
-        }),
+            InvalidReplicaAssignmentException::new),
     INVALID_CONFIG(40, "Configuration is invalid.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidConfigurationException(message);
-            }
-        }),
+            InvalidConfigurationException::new),
     NOT_CONTROLLER(41, "This is not the correct controller for this cluster.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new NotControllerException(message);
-            }
-        }),
+            NotControllerException::new),
     INVALID_REQUEST(42, "This most likely occurs because of a request being malformed by the " +
                 "client library or the message was sent to an incompatible broker. See the broker logs " +
                 "for more details.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidRequestException(message);
-            }
-        }),
+            InvalidRequestException::new),
     UNSUPPORTED_FOR_MESSAGE_FORMAT(43, "The message format version on the broker does not support the request.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnsupportedForMessageFormatException(message);
-            }
-        }),
+            UnsupportedForMessageFormatException::new),
     POLICY_VIOLATION(44, "Request parameters do not satisfy the configured policy.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new PolicyViolationException(message);
-            }
-        }),
+            PolicyViolationException::new),
     OUT_OF_ORDER_SEQUENCE_NUMBER(45, "The broker received an out of order sequence number",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new OutOfOrderSequenceException(message);
-            }
-        }),
+            OutOfOrderSequenceException::new),
     DUPLICATE_SEQUENCE_NUMBER(46, "The broker received a duplicate sequence number",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new DuplicateSequenceException(message);
-            }
-        }),
+            DuplicateSequenceException::new),
     INVALID_PRODUCER_EPOCH(47, "Producer attempted an operation with an old epoch. Either there is a newer producer " +
             "with the same transactionalId, or the producer's transaction has been expired by the broker.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ProducerFencedException(message);
-            }
-        }),
+            ProducerFencedException::new),
     INVALID_TXN_STATE(48, "The producer attempted a transactional operation in an invalid state",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidTxnStateException(message);
-            }
-        }),
+            InvalidTxnStateException::new),
     INVALID_PRODUCER_ID_MAPPING(49, "The producer attempted to use a producer id which is not currently assigned to " +
             "its transactional id",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidPidMappingException(message);
-            }
-        }),
+            InvalidPidMappingException::new),
     INVALID_TRANSACTION_TIMEOUT(50, "The transaction timeout is larger than the maximum value allowed by " +
                 "the broker (as configured by transaction.max.timeout.ms).",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidTxnTimeoutException(message);
-            }
-        }),
+            InvalidTxnTimeoutException::new),
     CONCURRENT_TRANSACTIONS(51, "The producer attempted to update a transaction " +
                 "while another concurrent operation on the same transaction was ongoing",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ConcurrentTransactionsException(message);
-            }
-        }),
+            ConcurrentTransactionsException::new),
     TRANSACTION_COORDINATOR_FENCED(52, "Indicates that the transaction coordinator sending a WriteTxnMarker " +
             "is no longer the current coordinator for a given producer",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new TransactionCoordinatorFencedException(message);
-            }
-        }),
+            TransactionCoordinatorFencedException::new),
     TRANSACTIONAL_ID_AUTHORIZATION_FAILED(53, "Transactional Id authorization failed",
-                                                  new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new TransactionalIdAuthorizationException(message);
-        }
-    }),
-    SECURITY_DISABLED(54, "Security features are disabled.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new SecurityDisabledException(message);
-        }
-    }),
-    OPERATION_NOT_ATTEMPTED(55, "The broker did not attempt to execute this operation. This may happen for batched RPCs " +
-            "where some operations in the batch failed, causing the broker to respond without trying the rest.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new OperationNotAttemptedException(message);
-            }
-    }),
+            TransactionalIdAuthorizationException::new),
+    SECURITY_DISABLED(54, "Security features are disabled.",
+            SecurityDisabledException::new),
+    OPERATION_NOT_ATTEMPTED(55, "The broker did not attempt to execute this operation. This may happen for " +
+            "batched RPCs where some operations in the batch failed, causing the broker to respond without " +
+            "trying the rest.",
+            OperationNotAttemptedException::new),
     KAFKA_STORAGE_ERROR(56, "Disk error when trying to access log file on the disk.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new KafkaStorageException(message);
-            }
-    }),
+            KafkaStorageException::new),
     LOG_DIR_NOT_FOUND(57, "The user-specified log directory is not found in the broker config.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new LogDirNotFoundException(message);
-            }
-    }),
+            LogDirNotFoundException::new),
     SASL_AUTHENTICATION_FAILED(58, "SASL Authentication failed.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new SaslAuthenticationException(message);
-            }
-    }),
+            SaslAuthenticationException::new),
     UNKNOWN_PRODUCER_ID(59, "This exception is raised by the broker if it could not locate the producer metadata " +
             "associated with the producerId in question. This could happen if, for instance, the producer's records " +
             "were deleted because their retention time had elapsed. Once the last records of the producerId are " +
             "removed, the producer's metadata is removed from the broker, and future appends by the producer will " +
             "return this exception.",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new UnknownProducerIdException(message);
-            }
-    }),
+            UnknownProducerIdException::new),
     REASSIGNMENT_IN_PROGRESS(60, "A partition reassignment is in progress",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ReassignmentInProgressException(message);
-            }
-        }),
-    DELEGATION_TOKEN_AUTH_DISABLED(61, "Delegation Token feature is not enabled.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new DelegationTokenDisabledException(message);
-        }
-    }),
-    DELEGATION_TOKEN_NOT_FOUND(62, "Delegation Token is not found on server.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new DelegationTokenNotFoundException(message);
-        }
-    }),
-    DELEGATION_TOKEN_OWNER_MISMATCH(63, "Specified Principal is not valid Owner/Renewer.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new DelegationTokenOwnerMismatchException(message);
-        }
-    }),
-    DELEGATION_TOKEN_REQUEST_NOT_ALLOWED(64, "Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and " + "on delegation token authenticated channels.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new UnsupportedByAuthenticationException(message);
-        }
-    }),
-    DELEGATION_TOKEN_AUTHORIZATION_FAILED(65, "Delegation Token authorization failed.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new DelegationTokenAuthorizationException(message);
-        }
-    }),
-    DELEGATION_TOKEN_EXPIRED(66, "Delegation Token is expired.", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new DelegationTokenExpiredException(message);
-        }
-    }),
-    INVALID_PRINCIPAL_TYPE(67, "Supplied principalType is not supported", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new InvalidPrincipalTypeException(message);
-        }
-    }),
-    NON_EMPTY_GROUP(68, "The group is not empty", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new GroupNotEmptyException(message);
-        }
-    }),
-    GROUP_ID_NOT_FOUND(69, "The group id does not exist", new ApiExceptionBuilder() {
-        @Override
-        public ApiException build(String message) {
-            return new GroupIdNotFoundException(message);
-        }
-    }),
+            ReassignmentInProgressException::new),
+    DELEGATION_TOKEN_AUTH_DISABLED(61, "Delegation Token feature is not enabled.",
+            DelegationTokenDisabledException::new),
+    DELEGATION_TOKEN_NOT_FOUND(62, "Delegation Token is not found on server.",
+            DelegationTokenNotFoundException::new),
+    DELEGATION_TOKEN_OWNER_MISMATCH(63, "Specified Principal is not valid Owner/Renewer.",
+            DelegationTokenOwnerMismatchException::new),
+    DELEGATION_TOKEN_REQUEST_NOT_ALLOWED(64, "Delegation Token requests are not allowed on PLAINTEXT/1-way SSL " +
+            "channels and on delegation token authenticated channels.",
+            UnsupportedByAuthenticationException::new),
+    DELEGATION_TOKEN_AUTHORIZATION_FAILED(65, "Delegation Token authorization failed.",
+            DelegationTokenAuthorizationException::new),
+    DELEGATION_TOKEN_EXPIRED(66, "Delegation Token is expired.",
+            DelegationTokenExpiredException::new),
+    INVALID_PRINCIPAL_TYPE(67, "Supplied principalType is not supported",
+            InvalidPrincipalTypeException::new),
+    NON_EMPTY_GROUP(68, "The group is not empty",
+            GroupNotEmptyException::new),
+    GROUP_ID_NOT_FOUND(69, "The group id does not exist",
+            GroupIdNotFoundException::new),
     FETCH_SESSION_ID_NOT_FOUND(70, "The fetch session ID was not found",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new FetchSessionIdNotFoundException(message);
-            }
-    }),
+            FetchSessionIdNotFoundException::new),
     INVALID_FETCH_SESSION_EPOCH(71, "The fetch session epoch is invalid",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new InvalidFetchSessionEpochException(message);
-            }
-    }),
-    LISTENER_NOT_FOUND(72, "There is no listener on the leader broker that matches the listener on which metadata request was processed",
-        new ApiExceptionBuilder() {
-            @Override
-            public ApiException build(String message) {
-                return new ListenerNotFoundException(message);
-            }
-    }),;
-
-    private interface ApiExceptionBuilder {
-        ApiException build(String message);
-    }
+            InvalidFetchSessionEpochException::new),
+    LISTENER_NOT_FOUND(72, "There is no listener on the leader broker that matches the listener on which " +
+            "metadata request was processed",
+            ListenerNotFoundException::new),;
 
     private static final Logger log = LoggerFactory.getLogger(Errors.class);
 
@@ -652,13 +291,13 @@ private interface ApiExceptionBuilder {
     }
 
     private final short code;
-    private final ApiExceptionBuilder builder;
+    private final Function builder;
     private final ApiException exception;
 
-    Errors(int code, String defaultExceptionString, ApiExceptionBuilder builder) {
+    Errors(int code, String defaultExceptionString, Function builder) {
         this.code = (short) code;
         this.builder = builder;
-        this.exception = builder.build(defaultExceptionString);
+        this.exception = builder.apply(defaultExceptionString);
     }
 
     /**
@@ -680,7 +319,7 @@ public ApiException exception(String message) {
             return exception;
         }
         // Return an exception with the given error message.
-        return builder.build(message);
+        return builder.apply(message);
     }
 
     /**

From b1090e52a3c7ea83ef2d79cc4b6ff7084a5070a4 Mon Sep 17 00:00:00 2001
From: Viktor Somogyi 
Date: Thu, 23 Aug 2018 15:40:33 +0200
Subject: [PATCH 0739/1847] MINOR: Eliminate warnings from KafkaProducerTest
 (#5548)

And code clean-ups in the same file.

Reviewers: Kamal Chandraprakash , Ismael Juma 
---
 .../clients/producer/KafkaProducerTest.java   | 145 ++++++++----------
 1 file changed, 67 insertions(+), 78 deletions(-)

diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
index dd2dd896b2855..22fa0a1bfc0f4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
@@ -27,7 +27,6 @@
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.errors.InterruptException;
 import org.apache.kafka.common.errors.InvalidTopicException;
@@ -45,7 +44,6 @@
 import org.apache.kafka.test.MockPartitioner;
 import org.apache.kafka.test.MockProducerInterceptor;
 import org.apache.kafka.test.MockSerializer;
-import org.apache.kafka.test.TestCondition;
 import org.apache.kafka.test.TestUtils;
 import org.easymock.EasyMock;
 import org.junit.Assert;
@@ -100,7 +98,7 @@ public void testConstructorFailureCloseResource() {
 
         final int oldInitCount = MockMetricsReporter.INIT_COUNT.get();
         final int oldCloseCount = MockMetricsReporter.CLOSE_COUNT.get();
-        try (KafkaProducer producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) {
+        try (KafkaProducer ignored = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) {
             fail("should have caught an exception and returned");
         } catch (KafkaException e) {
             assertEquals(oldInitCount + 1, MockMetricsReporter.INIT_COUNT.get());
@@ -110,7 +108,7 @@ public void testConstructorFailureCloseResource() {
     }
 
     @Test
-    public void testSerializerClose() throws Exception {
+    public void testSerializerClose() {
         Map configs = new HashMap<>();
         configs.put(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose");
         configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
@@ -119,7 +117,7 @@ public void testSerializerClose() throws Exception {
         final int oldInitCount = MockSerializer.INIT_COUNT.get();
         final int oldCloseCount = MockSerializer.CLOSE_COUNT.get();
 
-        KafkaProducer producer = new KafkaProducer(
+        KafkaProducer producer = new KafkaProducer<>(
                 configs, new MockSerializer(), new MockSerializer());
         assertEquals(oldInitCount + 2, MockSerializer.INIT_COUNT.get());
         assertEquals(oldCloseCount, MockSerializer.CLOSE_COUNT.get());
@@ -130,7 +128,7 @@ public void testSerializerClose() throws Exception {
     }
 
     @Test
-    public void testInterceptorConstructClose() throws Exception {
+    public void testInterceptorConstructClose() {
         try {
             Properties props = new Properties();
             // test with client ID assigned by KafkaProducer
@@ -138,7 +136,7 @@ public void testInterceptorConstructClose() throws Exception {
             props.setProperty(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, MockProducerInterceptor.class.getName());
             props.setProperty(MockProducerInterceptor.APPEND_STRING_PROP, "something");
 
-            KafkaProducer producer = new KafkaProducer(
+            KafkaProducer producer = new KafkaProducer<>(
                     props, new StringSerializer(), new StringSerializer());
             assertEquals(1, MockProducerInterceptor.INIT_COUNT.get());
             assertEquals(0, MockProducerInterceptor.CLOSE_COUNT.get());
@@ -156,14 +154,14 @@ public void testInterceptorConstructClose() throws Exception {
     }
 
     @Test
-    public void testPartitionerClose() throws Exception {
+    public void testPartitionerClose() {
         try {
             Properties props = new Properties();
             props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
             MockPartitioner.resetCounters();
             props.setProperty(ProducerConfig.PARTITIONER_CLASS_CONFIG, MockPartitioner.class.getName());
 
-            KafkaProducer producer = new KafkaProducer(
+            KafkaProducer producer = new KafkaProducer<>(
                     props, new StringSerializer(), new StringSerializer());
             assertEquals(1, MockPartitioner.INIT_COUNT.get());
             assertEquals(0, MockPartitioner.CLOSE_COUNT.get());
@@ -189,7 +187,7 @@ public void shouldCloseProperlyAndThrowIfInterrupted() throws Exception {
         Node node = cluster.nodes().get(0);
 
         Metadata metadata = new Metadata(0, Long.MAX_VALUE, true);
-        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
 
         MockClient client = new MockClient(time, metadata);
         client.setNode(node);
@@ -201,16 +199,13 @@ public void shouldCloseProperlyAndThrowIfInterrupted() throws Exception {
         ExecutorService executor = Executors.newSingleThreadExecutor();
         final AtomicReference closeException = new AtomicReference<>();
         try {
-            Future future = executor.submit(new Runnable() {
-                @Override
-                public void run() {
-                    producer.send(new ProducerRecord<>("topic", "key", "value"));
-                    try {
-                        producer.close();
-                        fail("Close should block and throw.");
-                    } catch (Exception e) {
-                        closeException.set(e);
-                    }
+            Future future = executor.submit(() -> {
+                producer.send(new ProducerRecord<>("topic", "key", "value"));
+                try {
+                    producer.close();
+                    fail("Close should block and throw.");
+                } catch (Exception e) {
+                    closeException.set(e);
                 }
             });
 
@@ -225,14 +220,11 @@ public void run() {
 
             assertTrue("Close terminated prematurely", future.cancel(true));
 
-            TestUtils.waitForCondition(new TestCondition() {
-                @Override
-                public boolean conditionMet() {
-                    return closeException.get() != null;
-                }
-            }, "InterruptException did not occur within timeout.");
+            TestUtils.waitForCondition(() -> closeException.get() != null,
+                    "InterruptException did not occur within timeout.");
 
-            assertTrue("Expected exception not thrown " + closeException, closeException.get() instanceof InterruptException);
+            assertTrue("Expected exception not thrown " + closeException,
+                    closeException.get() instanceof InterruptException);
         } finally {
             executor.shutdownNow();
         }
@@ -240,7 +232,7 @@ public boolean conditionMet() {
     }
 
     @Test
-    public void testOsDefaultSocketBufferSizes() throws Exception {
+    public void testOsDefaultSocketBufferSizes() {
         Map config = new HashMap<>();
         config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
         config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE);
@@ -249,7 +241,7 @@ public void testOsDefaultSocketBufferSizes() throws Exception {
     }
 
     @Test(expected = KafkaException.class)
-    public void testInvalidSocketSendBufferSize() throws Exception {
+    public void testInvalidSocketSendBufferSize() {
         Map config = new HashMap<>();
         config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
         config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2);
@@ -257,7 +249,7 @@ public void testInvalidSocketSendBufferSize() throws Exception {
     }
 
     @Test(expected = KafkaException.class)
-    public void testInvalidSocketReceiveBufferSize() throws Exception {
+    public void testInvalidSocketReceiveBufferSize() {
         Map config = new HashMap<>();
         config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
         config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2);
@@ -277,15 +269,15 @@ public void testMetadataFetch() throws Exception {
         ProducerRecord record = new ProducerRecord<>(topic, "value");
         Collection nodes = Collections.singletonList(new Node(0, "host1", 1000));
         final Cluster emptyCluster = new Cluster(null, nodes,
-                Collections.emptySet(),
-                Collections.emptySet(),
-                Collections.emptySet());
+                Collections.emptySet(),
+                Collections.emptySet(),
+                Collections.emptySet());
         final Cluster cluster = new Cluster(
                 "dummy",
                 Collections.singletonList(new Node(0, "host1", 1000)),
-                Arrays.asList(new PartitionInfo(topic, 0, null, null, null)),
-                Collections.emptySet(),
-                Collections.emptySet());
+                Collections.singletonList(new PartitionInfo(topic, 0, null, null, null)),
+                Collections.emptySet(),
+                Collections.emptySet());
 
         // Expect exactly one fetch for each attempt to refresh while topic metadata is not available
         final int refreshAttempts = 5;
@@ -328,15 +320,15 @@ public void testMetadataFetchOnStaleMetadata() throws Exception {
         ProducerRecord extendedRecord = new ProducerRecord<>(topic, 2, null, "value");
         Collection nodes = Collections.singletonList(new Node(0, "host1", 1000));
         final Cluster emptyCluster = new Cluster(null, nodes,
-                Collections.emptySet(),
-                Collections.emptySet(),
-                Collections.emptySet());
+                Collections.emptySet(),
+                Collections.emptySet(),
+                Collections.emptySet());
         final Cluster initialCluster = new Cluster(
                 "dummy",
                 Collections.singletonList(new Node(0, "host1", 1000)),
-                Arrays.asList(new PartitionInfo(topic, 0, null, null, null)),
-                Collections.emptySet(),
-                Collections.emptySet());
+                Collections.singletonList(new PartitionInfo(topic, 0, null, null, null)),
+                Collections.emptySet(),
+                Collections.emptySet());
         final Cluster extendedCluster = new Cluster(
                 "dummy",
                 Collections.singletonList(new Node(0, "host1", 1000)),
@@ -344,8 +336,8 @@ public void testMetadataFetchOnStaleMetadata() throws Exception {
                         new PartitionInfo(topic, 0, null, null, null),
                         new PartitionInfo(topic, 1, null, null, null),
                         new PartitionInfo(topic, 2, null, null, null)),
-                Collections.emptySet(),
-                Collections.emptySet());
+                Collections.emptySet(),
+                Collections.emptySet());
 
         // Expect exactly one fetch for each attempt to refresh while topic metadata is not available
         final int refreshAttempts = 5;
@@ -405,18 +397,15 @@ public void testTopicRefreshInMetadata() throws Exception {
         MemberModifier.field(KafkaProducer.class, "time").set(producer, time);
         final String topic = "topic";
 
-        Thread t = new Thread() {
-            @Override
-            public void run() {
-                long startTimeMs = System.currentTimeMillis();
-                for (int i = 0; i < 10; i++) {
-                    while (!metadata.updateRequested() && System.currentTimeMillis() - startTimeMs < 1000)
-                        yield();
-                    metadata.update(Cluster.empty(), Collections.singleton(topic), time.milliseconds());
-                    time.sleep(60 * 1000L);
-                }
+        Thread t = new Thread(() -> {
+            long startTimeMs = System.currentTimeMillis();
+            for (int i = 0; i < 10; i++) {
+                while (!metadata.updateRequested() && System.currentTimeMillis() - startTimeMs < 1000)
+                    Thread.yield();
+                metadata.update(Cluster.empty(), Collections.singleton(topic), time.milliseconds());
+                time.sleep(60 * 1000L);
             }
-        };
+        });
         t.start();
         try {
             producer.partitionsFor(topic);
@@ -432,8 +421,10 @@ public void run() {
     public void testHeaders() throws Exception {
         Properties props = new Properties();
         props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
-        ExtendedSerializer keySerializer = PowerMock.createNiceMock(ExtendedSerializer.class);
-        ExtendedSerializer valueSerializer = PowerMock.createNiceMock(ExtendedSerializer.class);
+        @SuppressWarnings("unchecked") // it is safe to suppress, since this is a mock class
+        ExtendedSerializer keySerializer = PowerMock.createNiceMock(ExtendedSerializer.class);
+        @SuppressWarnings("unchecked")
+        ExtendedSerializer valueSerializer = PowerMock.createNiceMock(ExtendedSerializer.class);
 
         KafkaProducer producer = new KafkaProducer<>(props, keySerializer, valueSerializer);
         Metadata metadata = PowerMock.createNiceMock(Metadata.class);
@@ -443,9 +434,9 @@ public void testHeaders() throws Exception {
         final Cluster cluster = new Cluster(
                 "dummy",
                 Collections.singletonList(new Node(0, "host1", 1000)),
-                Arrays.asList(new PartitionInfo(topic, 0, null, null, null)),
-                Collections.emptySet(),
-                Collections.emptySet());
+                Collections.singletonList(new PartitionInfo(topic, 0, null, null, null)),
+                Collections.emptySet(),
+                Collections.emptySet());
 
 
         EasyMock.expect(metadata.fetch()).andReturn(cluster).anyTimes();
@@ -522,15 +513,16 @@ public void testInterceptorPartitionSetOnTooLargeRecord() throws Exception {
         final Cluster cluster = new Cluster(
             "dummy",
             Collections.singletonList(new Node(0, "host1", 1000)),
-            Arrays.asList(new PartitionInfo(topic, 0, null, null, null)),
-            Collections.emptySet(),
-            Collections.emptySet());
+            Collections.singletonList(new PartitionInfo(topic, 0, null, null, null)),
+            Collections.emptySet(),
+            Collections.emptySet());
         EasyMock.expect(metadata.fetch()).andReturn(cluster).once();
 
         // Mock interceptors field
-        ProducerInterceptors interceptors = PowerMock.createMock(ProducerInterceptors.class);
+        @SuppressWarnings("unchecked") // it is safe to suppress, since this is a mock class
+        ProducerInterceptors interceptors = PowerMock.createMock(ProducerInterceptors.class);
         EasyMock.expect(interceptors.onSend(record)).andReturn(record);
-        interceptors.onSendError(EasyMock.eq(record), EasyMock.notNull(), EasyMock.notNull());
+        interceptors.onSendError(EasyMock.eq(record), EasyMock.notNull(), EasyMock.notNull());
         EasyMock.expectLastCall();
         MemberModifier.field(KafkaProducer.class, "interceptors").set(producer, interceptors);
 
@@ -565,19 +557,16 @@ public void testInitTransactionTimeout() {
         Node node = cluster.nodes().get(0);
 
         Metadata metadata = new Metadata(0, Long.MAX_VALUE, true);
-        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
 
         MockClient client = new MockClient(time, metadata);
         client.setNode(node);
 
-        Producer producer = new KafkaProducer<>(
+        try (Producer producer = new KafkaProducer<>(
                 new ProducerConfig(ProducerConfig.addSerializerToConfig(props, new StringSerializer(), new StringSerializer())),
-                new StringSerializer(), new StringSerializer(), metadata, client);
-        try {
+                new StringSerializer(), new StringSerializer(), metadata, client)) {
             producer.initTransactions();
             fail("initTransactions() should have raised TimeoutException");
-        } finally {
-            producer.close(0, TimeUnit.MILLISECONDS);
         }
     }
 
@@ -593,7 +582,7 @@ public void testOnlyCanExecuteCloseAfterInitTransactionsTimeout() {
         Node node = cluster.nodes().get(0);
 
         Metadata metadata = new Metadata(0, Long.MAX_VALUE, true);
-        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
 
         MockClient client = new MockClient(time, metadata);
         client.setNode(node);
@@ -626,7 +615,7 @@ public void testSendToInvalidTopic() throws Exception {
         Node node = cluster.nodes().get(0);
 
         Metadata metadata = new Metadata(0, Long.MAX_VALUE, true);
-        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
 
         MockClient client = new MockClient(time, metadata);
         client.setNode(node);
@@ -638,16 +627,16 @@ public void testSendToInvalidTopic() throws Exception {
         String invalidTopicName = "topic abc";          // Invalid topic name due to space
         ProducerRecord record = new ProducerRecord<>(invalidTopicName, "HelloKafka");
 
-        Set invalidTopic = new HashSet();
+        Set invalidTopic = new HashSet<>();
         invalidTopic.add(invalidTopicName);
         Cluster metaDataUpdateResponseCluster = new Cluster(cluster.clusterResource().clusterId(),
                 cluster.nodes(),
-                new ArrayList(0),
-                Collections.emptySet(),
+                new ArrayList<>(0),
+                Collections.emptySet(),
                 invalidTopic,
                 cluster.internalTopics(),
                 cluster.controller());
-        client.prepareMetadataUpdate(metaDataUpdateResponseCluster, Collections.emptySet());
+        client.prepareMetadataUpdate(metaDataUpdateResponseCluster, Collections.emptySet());
 
         Future future = producer.send(record);
 
@@ -669,7 +658,7 @@ public void testCloseWhenWaitingForMetadataUpdate() throws InterruptedException
         Cluster cluster = TestUtils.singletonCluster();
         Node node = cluster.nodes().get(0);
         Metadata metadata = new Metadata(0, Long.MAX_VALUE, false);
-        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
         MockClient client = new MockClient(time, metadata);
         client.setNode(node);
 

From d8f9f278a2326f3d1ef872a21ef1b43593df571f Mon Sep 17 00:00:00 2001
From: Joan Goyeau 
Date: Thu, 23 Aug 2018 17:15:27 +0100
Subject: [PATCH 0740/1847] KAFKA-7316: Fix Streams Scala filter recursive call
 #5538

Due to lack of conversion to kstream Predicate, existing filter method in KTable.scala would result in StackOverflowError.

This PR fixes the bug and adds testing for it.

Reviewers: Guozhang Wang , John Roesler 
---
 .../kafka/streams/scala/kstream/KTable.scala  |  4 +-
 .../kafka/streams/scala/KStreamTest.scala     | 48 ++++++++++++++
 .../kafka/streams/scala/KTableTest.scala      | 66 +++++++++++++++++++
 3 files changed, 116 insertions(+), 2 deletions(-)

diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
index a78d321c941ea..d41496fb21cd2 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KTable.scala
@@ -47,7 +47,7 @@ class KTable[K, V](val inner: KTableJ[K, V]) {
    * @see `org.apache.kafka.streams.kstream.KTable#filter`
    */
   def filter(predicate: (K, V) => Boolean): KTable[K, V] =
-    inner.filter(predicate(_, _))
+    inner.filter(predicate.asPredicate)
 
   /**
    * Create a new [[KTable]] that consists all records of this [[KTable]] which satisfies the given
@@ -71,7 +71,7 @@ class KTable[K, V](val inner: KTableJ[K, V]) {
    * @see `org.apache.kafka.streams.kstream.KTable#filterNot`
    */
   def filterNot(predicate: (K, V) => Boolean): KTable[K, V] =
-    inner.filterNot(predicate(_, _))
+    inner.filterNot(predicate.asPredicate)
 
   /**
    * Create a new [[KTable]] that consists all records of this [[KTable]] which do not satisfy the given
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
index 6a302b207a9cc..2e2132d14eb89 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
@@ -29,6 +29,52 @@ import org.scalatest.{FlatSpec, Matchers}
 @RunWith(classOf[JUnitRunner])
 class KStreamTest extends FlatSpec with Matchers with TestDriver {
 
+  "filter a KStream" should "filter records satisfying the predicate" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+    val sinkTopic = "sink"
+
+    builder.stream[String, String](sourceTopic).filter((_, value) => value != "value2").to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value1"
+
+    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
+    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+
+    testDriver.pipeRecord(sourceTopic, ("3", "value3"))
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value3"
+
+    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+
+  "filterNot a KStream" should "filter records not satisfying the predicate" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+    val sinkTopic = "sink"
+
+    builder.stream[String, String](sourceTopic).filterNot((_, value) => value == "value2").to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value1"
+
+    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
+    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+
+    testDriver.pipeRecord(sourceTopic, ("3", "value3"))
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value3"
+
+    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+
   "selectKey a KStream" should "select a new key" in {
     val builder = new StreamsBuilder()
     val sourceTopic = "source"
@@ -44,6 +90,8 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     testDriver.pipeRecord(sourceTopic, ("1", "value2"))
     testDriver.readRecord[String, String](sinkTopic).key shouldBe "value2"
 
+    testDriver.readRecord[String, String](sinkTopic) shouldBe null
+
     testDriver.close()
   }
 
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala
index 8c88ff5066f0e..2e9c821ed800c 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KTableTest.scala
@@ -29,6 +29,72 @@ import org.scalatest.{FlatSpec, Matchers}
 @RunWith(classOf[JUnitRunner])
 class KTableTest extends FlatSpec with Matchers with TestDriver {
 
+  "filter a KTable" should "filter records satisfying the predicate" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+    val sinkTopic = "sink"
+
+    val table = builder.stream[String, String](sourceTopic).groupBy((key, _) => key).count()
+    table.filter((_, value) => value > 1).toStream.to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    {
+      testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+      val record = testDriver.readRecord[String, Long](sinkTopic)
+      record.key shouldBe "1"
+      record.value shouldBe (null: java.lang.Long)
+    }
+    {
+      testDriver.pipeRecord(sourceTopic, ("1", "value2"))
+      val record = testDriver.readRecord[String, Long](sinkTopic)
+      record.key shouldBe "1"
+      record.value shouldBe 2
+    }
+    {
+      testDriver.pipeRecord(sourceTopic, ("2", "value1"))
+      val record = testDriver.readRecord[String, Long](sinkTopic)
+      record.key shouldBe "2"
+      record.value shouldBe (null: java.lang.Long)
+    }
+    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+
+  "filterNot a KTable" should "filter records not satisfying the predicate" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+    val sinkTopic = "sink"
+
+    val table = builder.stream[String, String](sourceTopic).groupBy((key, _) => key).count()
+    table.filterNot((_, value) => value > 1).toStream.to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    {
+      testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+      val record = testDriver.readRecord[String, Long](sinkTopic)
+      record.key shouldBe "1"
+      record.value shouldBe 1
+    }
+    {
+      testDriver.pipeRecord(sourceTopic, ("1", "value2"))
+      val record = testDriver.readRecord[String, Long](sinkTopic)
+      record.key shouldBe "1"
+      record.value shouldBe (null: java.lang.Long)
+    }
+    {
+      testDriver.pipeRecord(sourceTopic, ("2", "value1"))
+      val record = testDriver.readRecord[String, Long](sinkTopic)
+      record.key shouldBe "2"
+      record.value shouldBe 1
+    }
+    testDriver.readRecord[String, Long](sinkTopic) shouldBe null
+
+    testDriver.close()
+  }
+
   "join 2 KTables" should "join correctly records" in {
     val builder = new StreamsBuilder()
     val sourceTopic1 = "source1"

From c968267bf1362a91eca8bb22566da76d6c9aeb26 Mon Sep 17 00:00:00 2001
From: Kamal Chandraprakash 
Date: Fri, 24 Aug 2018 02:09:35 +0530
Subject: [PATCH 0741/1847] MINOR: Return correct instance of
 SessionWindowSerde (#5546)

Plus minor javadoc cleanups.

Reviewers: Matthias J. Sax ,Guozhang Wang , John Roesler 
---
 .../org/apache/kafka/streams/Topology.java    | 29 ++++--------
 .../kafka/streams/kstream/Consumed.java       |  2 +-
 .../kafka/streams/kstream/KGroupedStream.java |  8 ++--
 .../kafka/streams/kstream/KGroupedTable.java  | 23 ++++-----
 .../apache/kafka/streams/kstream/KStream.java |  8 ++--
 .../kafka/streams/kstream/Transformer.java    |  6 +--
 .../streams/kstream/ValueTransformer.java     |  4 +-
 .../apache/kafka/streams/kstream/Window.java  |  4 +-
 .../kafka/streams/kstream/WindowedSerdes.java |  2 +-
 .../apache/kafka/streams/kstream/Windows.java |  2 +-
 .../streams/processor/internals/Task.java     |  2 +-
 .../streams/kstream/WindowedSerdesTest.java   | 47 +++++++++++++++++++
 12 files changed, 86 insertions(+), 51 deletions(-)
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/WindowedSerdesTest.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/Topology.java b/streams/src/main/java/org/apache/kafka/streams/Topology.java
index 753185c216463..8b2a46b15c009 100644
--- a/streams/src/main/java/org/apache/kafka/streams/Topology.java
+++ b/streams/src/main/java/org/apache/kafka/streams/Topology.java
@@ -289,7 +289,7 @@ public synchronized Topology addSource(final String name,
      * Add a new source that consumes from topics matching the given pattern and forwards the records to child processor
      * and/or sink nodes.
      * The source will use the specified key and value deserializers.
-     * The provided de-/serializers will be used for all matched topics, so care should be taken to specify patterns for
+     * The provided de-/serializers will be used for all the specified topics, so care should be taken when specifying
      * topics that share the same key-value data format.
      *
      * @param offsetReset        the auto offset reset policy to use for this stream if no committed offsets found;
@@ -412,8 +412,7 @@ public synchronized Topology addSource(final AutoOffsetReset offsetReset,
      * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume
      * and write to its topic
      * @return itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, StreamPartitioner, String...)
      * @see #addSink(String, String, Serializer, Serializer, String...)
      * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...)
@@ -445,8 +444,7 @@ public synchronized Topology addSink(final String name,
      * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume
      * and write to its topic
      * @return itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, String...)
      * @see #addSink(String, String, Serializer, Serializer, String...)
      * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...)
@@ -474,8 +472,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume
      * and write to its topic
      * @return itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, String...)
      * @see #addSink(String, String, StreamPartitioner, String...)
      * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...)
@@ -505,8 +502,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume
      * and write to its topic
      * @return itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, String...)
      * @see #addSink(String, String, StreamPartitioner, String...)
      * @see #addSink(String, String, Serializer, Serializer, String...)
@@ -533,8 +529,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames       the name of one or more source or processor nodes whose output records this sink should consume
      *                          and dynamically write to topics
      * @return                  itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, StreamPartitioner, String...)
      * @see #addSink(String, String, Serializer, Serializer, String...)
      * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...)
@@ -567,8 +562,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames       the name of one or more source or processor nodes whose output records this sink should consume
      *                          and dynamically write to topics
      * @return                  itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, String...)
      * @see #addSink(String, String, Serializer, Serializer, String...)
      * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...)
@@ -597,8 +591,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames       the name of one or more source or processor nodes whose output records this sink should consume
      *                          and dynamically write to topics
      * @return                  itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, String...)
      * @see #addSink(String, String, StreamPartitioner, String...)
      * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...)
@@ -629,8 +622,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames       the name of one or more source or processor nodes whose output records this sink should consume
      *                          and dynamically write to topics
      * @return                  itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      * @see #addSink(String, String, String...)
      * @see #addSink(String, String, StreamPartitioner, String...)
      * @see #addSink(String, String, Serializer, Serializer, String...)
@@ -655,8 +647,7 @@ public synchronized  Topology addSink(final String name,
      * @param parentNames the name of one or more source or processor nodes whose output records this processor should receive
      * and process
      * @return itself
-     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name,
-     *                           or if this processor's name is equal to the parent's name
+     * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name
      */
     public synchronized Topology addProcessor(final String name,
                                               final ProcessorSupplier supplier,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Consumed.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Consumed.java
index e2132ec79a0c4..0af7dbea7b4ac 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Consumed.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Consumed.java
@@ -97,7 +97,7 @@ public static  Consumed with(final Serde keySerde,
     /**
      * Create an instance of {@link Consumed} with key and value {@link Serde}s.
      *
-     * @param keySerde   the key serde. If {@code null}the default key serde from config will be used
+     * @param keySerde   the key serde. If {@code null} the default key serde from config will be used
      * @param valueSerde the value serde. If {@code null} the default value serde from config will be used
      * @param         key type
      * @param         value type
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
index 53a2be79add13..7b69e0317ea34 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
@@ -126,7 +126,7 @@ public interface KGroupedStream {
      * aggregate and the record's value.
      * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
      * value as-is.
-     * Thus, {@code reduce(Reducer, String)} can be used to compute aggregate functions like sum, min, or max.
+     * Thus, {@code reduce(Reducer)} can be used to compute aggregate functions like sum, min, or max.
      * 
      
      * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
      * the same key.
@@ -189,7 +189,7 @@ public interface KGroupedStream {
      * 
      {@code
      * KafkaStreams streams = ... // compute sum
      * String queryableStoreName = "storeName" // the store name should be the name of the store as defined by the Materialized instance
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-key";
      * Long sumForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
      
@@ -271,7 +271,7 @@  KTable aggregate(final Initializer initializer,
      * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
      * aggregate (or for the very first record using the intermediate aggregation result provided via the
      * {@link Initializer}) and the record's value.
-     * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like
+     * Thus, {@code aggregate(Initializer, Aggregator, Materialized)} can be used to compute aggregate functions like
      * count (c.f. {@link #count()}).
      * 
      
      * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
@@ -286,7 +286,7 @@  KTable aggregate(final Initializer initializer,
      * 
      {@code
      * KafkaStreams streams = ... // some aggregation on value type double
      * String queryableStoreName = "storeName" // the store name should be the name of the store as defined by the Materialized instance
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-key";
      * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
      
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
index 0e263362f5ced..30f348c2eaf34 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
@@ -60,7 +60,7 @@ public interface KGroupedTable {
      * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
      * 
      {@code
      * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-word";
      * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
      
@@ -89,7 +89,6 @@ public interface KGroupedTable {
      * the same key into a new instance of {@link KTable}.
      * Records with {@code null} key are ignored.
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * that can be queried using the provided {@code queryableStoreName}.
      * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 
      
      * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
@@ -158,7 +157,7 @@ public interface KGroupedTable {
      * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
      * 
      {@code
      * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-word";
      * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
      
@@ -191,7 +190,6 @@ KTable reduce(final Reducer adder,
      * Combining implies that the type of the aggregate result is the same as the type of the input value
      * (c.f. {@link #aggregate(Initializer, Aggregator, Aggregator)}).
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * that can be queried using the provided {@code queryableStoreName}.
      * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 
      
      * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}.
@@ -202,7 +200,7 @@ KTable reduce(final Reducer adder,
      * record from the aggregate.
      * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's
      * value as-is.
-     * Thus, {@code reduce(Reducer, Reducer, String)} can be used to compute aggregate functions like sum.
+     * Thus, {@code reduce(Reducer, Reducer)} can be used to compute aggregate functions like sum.
      * For sum, the adder and subtractor would work as follows:
      * 
      {@code
      * public class SumAdder implements Reducer {
@@ -243,12 +241,12 @@ KTable reduce(final Reducer adder,
 
     /**
      * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper)
-     * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers.
+     * mapped} to the same key into a new instance of {@link KTable}.
      * Records with {@code null} key are ignored.
      * Aggregating is a generalization of {@link #reduce(Reducer, Reducer, Materialized) combining via reduce(...)} as it,
      * for example, allows the result to have a different type than the input values.
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * provided by the given {@code storeSupplier}.
+     * that can be queried using the provided {@code queryableStoreName}.
      * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 
      
      * The specified {@link Initializer} is applied once directly before the first input record is processed to
@@ -260,11 +258,11 @@ KTable reduce(final Reducer adder,
      * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable}
      * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced"
      * record from the aggregate.
-     * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, String)} can be used to compute aggregate functions
+     * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, Materialized)} can be used to compute aggregate functions
      * like sum.
      * For sum, the initializer, adder, and subtractor would work as follows:
      * 
      {@code
-     * // in this example, LongSerde.class must be set as default value serde in StreamsConfig
+     * // in this example, LongSerde.class must be set as value serde in Materialized#withValueSerde
      * public class SumInitializer implements Initializer {
      *   public Long apply() {
      *     return 0L;
@@ -277,7 +275,7 @@ KTable reduce(final Reducer adder,
      *   }
      * }
      *
-     * public class SumSubstractor implements Aggregator {
+     * public class SumSubtractor implements Aggregator {
      *   public Long apply(String key, Integer oldValue, Long aggregate) {
      *     return aggregate - oldValue;
      *   }
@@ -294,7 +292,7 @@ KTable reduce(final Reducer adder,
      * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
      * 
      {@code
      * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-word";
      * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
      
@@ -333,7 +331,6 @@  KTable aggregate(final Initializer initializer,
      * If the result value type does not match the {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value
      * serde} you should use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized)}.
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * provided by the given {@code storeSupplier}.
      * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 
      
      * The specified {@link Initializer} is applied once directly before the first input record is processed to
@@ -362,7 +359,7 @@  KTable aggregate(final Initializer initializer,
      *   }
      * }
      *
-     * public class SumSubstractor implements Aggregator {
+     * public class SumSubtractor implements Aggregator {
      *   public Long apply(String key, Integer oldValue, Long aggregate) {
      *     return aggregate - oldValue;
      *   }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
index b6cc544cc4dfa..ae3b28a35ce0e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
@@ -190,7 +190,7 @@ public interface KStream {
      * The provided {@link ValueMapperWithKey} is applied to each input record value and computes a new value for it.
      * Thus, an input record {@code } can be transformed into an output record {@code }.
      * This is a stateless record-by-record operation (cf.
-     * {@link #transformValues(ValueTransformerSupplier, String...)} for stateful value transformation).
+     * {@link #transformValues(ValueTransformerWithKeySupplier, String...)} for stateful value transformation).
      * 
      
      * The example below counts the number of tokens of key and value strings.
      * 
      {@code
@@ -317,7 +317,7 @@ public interface KStream {
      * stream (value type can be altered arbitrarily).
      * The provided {@link ValueMapperWithKey} is applied to each input record and computes zero or more output values.
      * Thus, an input record {@code } can be transformed into output records {@code , , ...}.
-     * This is a stateless record-by-record operation (cf. {@link #transformValues(ValueTransformerSupplier, String...)}
+     * This is a stateless record-by-record operation (cf. {@link #transformValues(ValueTransformerWithKeySupplier, String...)}
      * for stateful value transformation).
      * 
      
      * The example below splits input records {@code }, with key=1, containing sentences as values
@@ -440,8 +440,8 @@ public interface KStream {
      * This is equivalent to calling {@link #to(String, Produced) to(someTopic, Produced.with(keySerde, valueSerde)}
      * and {@link StreamsBuilder#stream(String, Consumed) StreamsBuilder#stream(someTopicName, Consumed.with(keySerde, valueSerde))}.
      *
-     * @param topic
-     * @param produced
+     * @param topic     the topic name
+     * @param produced  the options to use when producing to the topic
      * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream}
      */
     KStream through(final String topic,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Transformer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Transformer.java
index 43b6115505eaf..0ab34699cf70a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Transformer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Transformer.java
@@ -52,7 +52,7 @@ public interface Transformer {
      * Initialize this transformer.
      * This is called once per instance when the topology gets initialized.
      * When the framework is done with the transformer, {@link #close()} will be called on it; the
-     * framework may later re-use the transformer by calling {@link #init()} again.
+     * framework may later re-use the transformer by calling {@link #init(ProcessorContext)} again.
      * 
      
      * The provided {@link ProcessorContext context} can be used to access topology and record meta data, to
      * {@link ProcessorContext#schedule(long, PunctuationType, Punctuator) schedule} a method to be
@@ -73,7 +73,7 @@ public interface Transformer {
      * 
      
      * If more than one output record should be forwarded downstream {@link ProcessorContext#forward(Object, Object)}
      * and {@link ProcessorContext#forward(Object, Object, To)} can be used.
-     * If not record should be forwarded downstream, {@code transform} can return {@code null}.
+     * If record should not be forwarded downstream, {@code transform} can return {@code null}.
      *
      * @param key the key for the record
      * @param value the value for the record
@@ -84,7 +84,7 @@ public interface Transformer {
 
     /**
      * Close this transformer and clean up any resources. The framework may
-     * later re-use this transformer by calling {@link #init()} on it again.
+     * later re-use this transformer by calling {@link #init(ProcessorContext)} on it again.
      * 
      
      * To generate new {@link KeyValue} pairs {@link ProcessorContext#forward(Object, Object)} and
      * {@link ProcessorContext#forward(Object, Object, To)} can be used.
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/ValueTransformer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/ValueTransformer.java
index 866cce8936182..b02311bf95890 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/ValueTransformer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/ValueTransformer.java
@@ -51,7 +51,7 @@ public interface ValueTransformer {
      * Initialize this transformer.
      * This is called once per instance when the topology gets initialized.
      * When the framework is done with the transformer, {@link #close()} will be called on it; the
-     * framework may later re-use the transformer by calling {@link #init()} again.
+     * framework may later re-use the transformer by calling {@link #init(ProcessorContext)} again.
      * 
      
      * The provided {@link ProcessorContext context} can be used to access topology and record meta data, to
      * {@link ProcessorContext#schedule(long, PunctuationType, Punctuator) schedule} a method to be
@@ -87,7 +87,7 @@ public interface ValueTransformer {
 
     /**
      * Close this transformer and clean up any resources. The framework may
-     * later re-use this transformer by calling {@link #init()} on it again.
+     * later re-use this transformer by calling {@link #init(ProcessorContext)} on it again.
      * 
      
      * It is not possible to return any new output records within {@code close()}.
      * Using {@link ProcessorContext#forward(Object, Object)} or {@link ProcessorContext#forward(Object, Object, To)}
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
index 08540a1aadfaa..f6250683282c8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
@@ -103,8 +103,8 @@ public int hashCode() {
     @Override
     public String toString() {
         return "Window{" +
-            "start=" + startMs +
-            ", end=" + endMs +
+            "startMs=" + startMs +
+            ", endMs=" + endMs +
             '}';
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/WindowedSerdes.java b/streams/src/main/java/org/apache/kafka/streams/kstream/WindowedSerdes.java
index d0381c787c05a..6a851a10d9d38 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/WindowedSerdes.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/WindowedSerdes.java
@@ -54,6 +54,6 @@ static public  Serde> timeWindowedSerdeFrom(final Class type)
      * Construct a {@code SessionWindowedSerde} object for the specified inner class type.
      */
     static public  Serde> sessionWindowedSerdeFrom(final Class type) {
-        return new TimeWindowedSerde<>(Serdes.serdeFrom(type));
+        return new SessionWindowedSerde<>(Serdes.serdeFrom(type));
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
index 406e44f73d417..8c9ba55f88daf 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
@@ -24,7 +24,7 @@
 import java.util.Objects;
 
 /**
- * The window specification interface for fixed size windows that is used to define window boundaries and grace period.
+ * The window specification for fixed size windows that is used to define window boundaries and grace period.
  *
  * Grace period defines how long to wait on late events, where lateness is defined as (stream_time - record_timestamp).
  *
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
index 5f221e3dc02e1..2b43640f00e31 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
@@ -27,7 +27,7 @@
 
 public interface Task {
     /**
-     * Initialize the task and return {}true if the task is ready to run, i.e, it has not state stores
+     * Initialize the task and return {@code true} if the task is ready to run, i.e, it has not state stores
      * @return true if this task has no state stores that may need restoring.
      * @throws IllegalStateException If store gets registered after initialized is already finished
      * @throws StreamsException if the store's change log does not contain the partition
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/WindowedSerdesTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/WindowedSerdesTest.java
new file mode 100644
index 0000000000000..4360d0846ec1f
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/WindowedSerdesTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.streams.kstream.internals.SessionWindow;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class WindowedSerdesTest {
+
+    private final String topic = "sample";
+
+    @Test
+    public void testTimeWindowSerdeFrom() {
+        final Windowed timeWindowed = new Windowed<>(10, new TimeWindow(0, Long.MAX_VALUE));
+        final Serde> timeWindowedSerde = WindowedSerdes.timeWindowedSerdeFrom(Integer.class);
+        final byte[] bytes = timeWindowedSerde.serializer().serialize(topic, timeWindowed);
+        final Windowed windowed = timeWindowedSerde.deserializer().deserialize(topic, bytes);
+        Assert.assertEquals(timeWindowed, windowed);
+    }
+
+    @Test
+    public void testSessionWindowedSerdeFrom() {
+        final Windowed sessionWindowed = new Windowed<>(10, new SessionWindow(0, 1));
+        final Serde> sessionWindowedSerde = WindowedSerdes.sessionWindowedSerdeFrom(Integer.class);
+        final byte[] bytes = sessionWindowedSerde.serializer().serialize(topic, sessionWindowed);
+        final Windowed windowed = sessionWindowedSerde.deserializer().deserialize(topic, bytes);
+        Assert.assertEquals(sessionWindowed, windowed);
+    }
+
+}

From e577f6d36665012c65fc651ad392dc49e87618e6 Mon Sep 17 00:00:00 2001
From: Randall Hauch 
Date: Thu, 23 Aug 2018 14:22:09 -0700
Subject: [PATCH 0742/1847] KAFKA-7225; Corrected system tests by generating
 external properties file (#5489)

Fix system tests from earlier #5445 by moving to the `ConnectSystemBase` class the creation & cleanup of a file that can be used as externalized secrets in connector configs.

Reviewers: Arjun Satish , Robert Yokota , Konstantine Karantasis , Jason Gustafson 
---
 tests/kafkatest/services/connect.py           | 19 ++++++++++++++++++-
 tests/kafkatest/tests/connect/connect_test.py | 13 +++++++------
 .../templates/connect-standalone.properties   |  3 +++
 3 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/tests/kafkatest/services/connect.py b/tests/kafkatest/services/connect.py
index 19beddd1b78ec..d8c8d5a7e80d5 100644
--- a/tests/kafkatest/services/connect.py
+++ b/tests/kafkatest/services/connect.py
@@ -40,6 +40,7 @@ class ConnectServiceBase(KafkaPathResolverMixin, Service):
     STDERR_FILE = os.path.join(PERSISTENT_ROOT, "connect.stderr")
     LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "connect-log4j.properties")
     PID_FILE = os.path.join(PERSISTENT_ROOT, "connect.pid")
+    EXTERNAL_CONFIGS_FILE = os.path.join(PERSISTENT_ROOT, "connect-external-configs.properties")
     CONNECT_REST_PORT = 8083
 
     # Currently the Connect worker supports waiting on three modes:
@@ -69,6 +70,7 @@ def __init__(self, context, num_nodes, kafka, files):
         self.files = files
         self.startup_mode = self.STARTUP_MODE_LISTEN
         self.environment = {}
+        self.external_config_template_func = None
 
     def pids(self, node):
         """Return process ids for Kafka Connect processes."""
@@ -87,6 +89,17 @@ def set_configs(self, config_template_func, connector_config_templates=None):
         self.config_template_func = config_template_func
         self.connector_config_templates = connector_config_templates
 
+    def set_external_configs(self, external_config_template_func):
+        """
+        Set the properties that will be written in the external file properties
+        as used by the org.apache.kafka.common.config.provider.FileConfigProvider.
+        When this is used, the worker configuration must also enable the FileConfigProvider.
+        This is not provided in the constructor in case the worker
+        config generally needs access to ZK/Kafka services to
+        create the configuration.
+        """
+        self.external_config_template_func = external_config_template_func
+
     def listening(self, node):
         try:
             cmd = "nc -z %s %s" % (node.account.hostname, self.CONNECT_REST_PORT)
@@ -145,7 +158,7 @@ def restart_node(self, node, clean_shutdown=True):
     def clean_node(self, node):
         node.account.kill_process("connect", clean_shutdown=False, allow_fail=True)
         self.security_config.clean_node(node)
-        all_files = " ".join([self.CONFIG_FILE, self.LOG4J_CONFIG_FILE, self.PID_FILE, self.LOG_FILE, self.STDOUT_FILE, self.STDERR_FILE] + self.config_filenames() + self.files)
+        all_files = " ".join([self.CONFIG_FILE, self.LOG4J_CONFIG_FILE, self.PID_FILE, self.LOG_FILE, self.STDOUT_FILE, self.STDERR_FILE, self.EXTERNAL_CONFIGS_FILE] + self.config_filenames() + self.files)
         node.account.ssh("rm -rf " + all_files, allow_fail=False)
 
     def config_filenames(self):
@@ -263,6 +276,8 @@ def start_node(self, node):
         node.account.ssh("mkdir -p %s" % self.PERSISTENT_ROOT, allow_fail=False)
 
         self.security_config.setup_node(node)
+        if self.external_config_template_func:
+            node.account.create_file(self.EXTERNAL_CONFIGS_FILE, self.external_config_template_func(node))
         node.account.create_file(self.CONFIG_FILE, self.config_template_func(node))
         node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('connect_log4j.properties', log_file=self.LOG_FILE))
         remote_connector_configs = []
@@ -308,6 +323,8 @@ def start_node(self, node):
         node.account.ssh("mkdir -p %s" % self.PERSISTENT_ROOT, allow_fail=False)
 
         self.security_config.setup_node(node)
+        if self.external_config_template_func:
+            node.account.create_file(self.EXTERNAL_CONFIGS_FILE, self.external_config_template_func(node))
         node.account.create_file(self.CONFIG_FILE, self.config_template_func(node))
         node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('connect_log4j.properties', log_file=self.LOG_FILE))
         if self.connector_config_templates:
diff --git a/tests/kafkatest/tests/connect/connect_test.py b/tests/kafkatest/tests/connect/connect_test.py
index c961681bd3ef8..e2618e9566133 100644
--- a/tests/kafkatest/tests/connect/connect_test.py
+++ b/tests/kafkatest/tests/connect/connect_test.py
@@ -47,7 +47,7 @@ class ConnectStandaloneFileTest(Test):
 
     OFFSETS_FILE = "/mnt/connect.offsets"
 
-    TOPIC = "${file:/mnt/connect/connect-file-external.properties:topic.external}"
+    TOPIC = "${file:" + EXTERNAL_CONFIGS_FILE + ":topic.external}"
     TOPIC_TEST = "test"
 
     FIRST_INPUT_LIST = ["foo", "bar", "baz"]
@@ -100,14 +100,12 @@ def test_file_source_and_sink(self, converter="org.apache.kafka.connect.json.Jso
         self.zk.start()
         self.kafka.start()
 
-        source_external_props = os.path.join(self.source.PERSISTENT_ROOT, "connect-file-external.properties")
-        self.source.node.account.create_file(source_external_props, self.render('connect-file-external.properties'))
         self.source.set_configs(lambda node: self.render("connect-standalone.properties", node=node), [self.render("connect-file-source.properties")])
-
-        sink_external_props = os.path.join(self.sink.PERSISTENT_ROOT, "connect-file-external.properties")
-        self.sink.node.account.create_file(sink_external_props, self.render('connect-file-external.properties'))
         self.sink.set_configs(lambda node: self.render("connect-standalone.properties", node=node), [self.render("connect-file-sink.properties")])
 
+        self.source.set_external_configs(lambda node: self.render("connect-file-external.properties", node=node))
+        self.sink.set_external_configs(lambda node: self.render("connect-file-external.properties", node=node))
+
         self.source.start()
         self.sink.start()
 
@@ -182,6 +180,9 @@ def test_skip_and_log_to_dlq(self, error_tolerance):
         self.override_value_converter_schemas_enable = False
         self.sink.set_configs(lambda node: self.render("connect-standalone.properties", node=node), [self.render("connect-file-sink.properties")])
 
+        self.source.set_external_configs(lambda node: self.render("connect-file-external.properties", node=node))
+        self.sink.set_external_configs(lambda node: self.render("connect-file-external.properties", node=node))
+
         self.source.start()
         self.sink.start()
 
diff --git a/tests/kafkatest/tests/connect/templates/connect-standalone.properties b/tests/kafkatest/tests/connect/templates/connect-standalone.properties
index cbfe491068228..a471dd5879c47 100644
--- a/tests/kafkatest/tests/connect/templates/connect-standalone.properties
+++ b/tests/kafkatest/tests/connect/templates/connect-standalone.properties
@@ -32,5 +32,8 @@ offset.storage.file.filename={{ OFFSETS_FILE }}
 # Reduce the admin client request timeouts so that we don't wait the default 120 sec before failing to connect the admin client
 request.timeout.ms=30000
 
+# Allow connector configs to use externalized config values of the form:
+#   ${file:/mnt/connect/connect-external-configs.properties:topic.external}
+#
 config.providers=file
 config.providers.file.class=org.apache.kafka.common.config.provider.FileConfigProvider

From 7884258489a190f076b0d4bd85ca68dbf808a989 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Thu, 23 Aug 2018 22:55:40 +0100
Subject: [PATCH 0743/1847] MINOR: Make JAAS configurable via template
 variables in system tests (#5554)

Currently, the only way in system tests to add a new variable to the `jaas.conf` template file is to directly edit the path the config is constructed by adding new keyword arguments.
This wasn't necessarily a big problem, since you'd only need edit the `security_config.py` file as JAAS settings should come from the security settings.

Now, with the addition of [KIP-342](https://cwiki.apache.org/confluence/display/KAFKA/KIP-342%3A+Add+support+for+Custom+SASL+extensions+in+OAuthBearer+authentication), the OAuthBearer JAAS config supports arbitrary values in the form of SASL extensions. This patch exposes a more convenient API to overrides these values in system tests.

Reviewers: Jason Gustafson 
---
 tests/kafkatest/services/console_consumer.py  |  7 ++-
 .../services/security/security_config.py      | 44 ++++++++++++++-----
 .../kafkatest/services/verifiable_consumer.py |  9 +++-
 .../kafkatest/services/verifiable_producer.py |  8 +++-
 4 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/tests/kafkatest/services/console_consumer.py b/tests/kafkatest/services/console_consumer.py
index 9e9ff56926bbd..7e78ffb87b135 100644
--- a/tests/kafkatest/services/console_consumer.py
+++ b/tests/kafkatest/services/console_consumer.py
@@ -61,7 +61,7 @@ def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-gro
                  message_validator=None, from_beginning=True, consumer_timeout_ms=None, version=DEV_BRANCH,
                  client_id="console-consumer", print_key=False, jmx_object_names=None, jmx_attributes=None,
                  enable_systest_events=False, stop_timeout_sec=15, print_timestamp=False,
-                 isolation_level="read_uncommitted"):
+                 isolation_level="read_uncommitted", jaas_override_variables=None):
         """
         Args:
             context:                    standard context
@@ -82,6 +82,8 @@ def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-gro
                                         and the corresponding background thread to finish successfully.
             print_timestamp             if True, print each message's timestamp as well
             isolation_level             How to handle transactional messages.
+            jaas_override_variables     A dict of variables to be used in the jaas.conf template file
+
         """
         JmxMixin.__init__(self, num_nodes=num_nodes, jmx_object_names=jmx_object_names, jmx_attributes=(jmx_attributes or []),
                           root=ConsoleConsumer.PERSISTENT_ROOT)
@@ -113,6 +115,7 @@ def __init__(self, context, num_nodes, kafka, topic, group_id="test-consumer-gro
             assert version >= V_0_10_0_0
 
         self.print_timestamp = print_timestamp
+        self.jaas_override_variables = jaas_override_variables or {}
 
     def prop_file(self, node):
         """Return a string which can be used to create a configuration file appropriate for the given node."""
@@ -125,7 +128,7 @@ def prop_file(self, node):
 
         # Add security properties to the config. If security protocol is not specified,
         # use the default in the template properties.
-        self.security_config = self.kafka.security_config.client_config(prop_file, node)
+        self.security_config = self.kafka.security_config.client_config(prop_file, node, self.jaas_override_variables)
         self.security_config.setup_node(node)
 
         prop_file += str(self.security_config)
diff --git a/tests/kafkatest/services/security/security_config.py b/tests/kafkatest/services/security/security_config.py
index 89ffe01aee350..b2fa4897652f3 100644
--- a/tests/kafkatest/services/security/security_config.py
+++ b/tests/kafkatest/services/security/security_config.py
@@ -112,7 +112,7 @@ class SecurityConfig(TemplateRenderer):
 
     def __init__(self, context, security_protocol=None, interbroker_security_protocol=None,
                  client_sasl_mechanism=SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SASL_MECHANISM_GSSAPI,
-                 zk_sasl=False, template_props="", static_jaas_conf=True):
+                 zk_sasl=False, template_props="", static_jaas_conf=True, jaas_override_variables=None):
         """
         Initialize the security properties for the node and copy
         keystore and truststore to the remote node if the transport protocol 
@@ -156,15 +156,20 @@ def __init__(self, context, security_protocol=None, interbroker_security_protoco
             'sasl.mechanism.inter.broker.protocol' : interbroker_sasl_mechanism,
             'sasl.kerberos.service.name' : 'kafka'
         }
+        self.jaas_override_variables = jaas_override_variables or {}
 
-    def client_config(self, template_props="", node=None):
+    def client_config(self, template_props="", node=None, jaas_override_variables=None):
         # If node is not specified, use static jaas config which will be created later.
         # Otherwise use static JAAS configuration files with SASL_SSL and sasl.jaas.config
         # property with SASL_PLAINTEXT so that both code paths are tested by existing tests.
         # Note that this is an artibtrary choice and it is possible to run all tests with
         # either static or dynamic jaas config files if required.
         static_jaas_conf = node is None or (self.has_sasl and self.has_ssl)
-        return SecurityConfig(self.context, self.security_protocol, client_sasl_mechanism=self.client_sasl_mechanism, template_props=template_props, static_jaas_conf=static_jaas_conf)
+        return SecurityConfig(self.context, self.security_protocol,
+                              client_sasl_mechanism=self.client_sasl_mechanism,
+                              template_props=template_props,
+                              static_jaas_conf=static_jaas_conf,
+                              jaas_override_variables=jaas_override_variables)
 
     def enable_security_protocol(self, security_protocol):
         self.has_sasl = self.has_sasl or self.is_sasl(security_protocol)
@@ -179,15 +184,18 @@ def setup_sasl(self, node):
         node.account.ssh("mkdir -p %s" % SecurityConfig.CONFIG_DIR, allow_fail=False)
         jaas_conf_file = "jaas.conf"
         java_version = node.account.ssh_capture("java -version")
-        if any('IBM' in line for line in java_version):
-            is_ibm_jdk = True
-        else:
-            is_ibm_jdk = False
-        jaas_conf = self.render(jaas_conf_file,  node=node, is_ibm_jdk=is_ibm_jdk,
-                                SecurityConfig=SecurityConfig,
-                                client_sasl_mechanism=self.client_sasl_mechanism,
-                                enabled_sasl_mechanisms=self.enabled_sasl_mechanisms,
-                                static_jaas_conf=self.static_jaas_conf)
+
+        jaas_conf = self.render_jaas_config(
+            jaas_conf_file,
+            {
+                'node': node,
+                'is_ibm_jdk': any('IBM' in line for line in java_version),
+                'SecurityConfig': SecurityConfig,
+                'client_sasl_mechanism': self.client_sasl_mechanism,
+                'enabled_sasl_mechanisms': self.enabled_sasl_mechanisms
+            }
+        )
+
         if self.static_jaas_conf:
             node.account.create_file(SecurityConfig.JAAS_CONF_PATH, jaas_conf)
         else:
@@ -196,6 +204,18 @@ def setup_sasl(self, node):
             node.account.copy_to(MiniKdc.LOCAL_KEYTAB_FILE, SecurityConfig.KEYTAB_PATH)
             node.account.copy_to(MiniKdc.LOCAL_KRB5CONF_FILE, SecurityConfig.KRB5CONF_PATH)
 
+    def render_jaas_config(self, jaas_conf_file, config_variables):
+        """
+        Renders the JAAS config file contents
+
+        :param jaas_conf_file: name of the JAAS config template file
+        :param config_variables: dict of variables used in the template
+        :return: the rendered template string
+        """
+        variables = config_variables.copy()
+        variables.update(self.jaas_override_variables)  # override variables
+        return self.render(jaas_conf_file, **variables)
+
     def setup_node(self, node):
         if self.has_ssl:
             self.setup_ssl(node)
diff --git a/tests/kafkatest/services/verifiable_consumer.py b/tests/kafkatest/services/verifiable_consumer.py
index 95970d99becba..a48afcf2b0e53 100644
--- a/tests/kafkatest/services/verifiable_consumer.py
+++ b/tests/kafkatest/services/verifiable_consumer.py
@@ -161,7 +161,10 @@ class VerifiableConsumer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou
     def __init__(self, context, num_nodes, kafka, topic, group_id,
                  max_messages=-1, session_timeout_sec=30, enable_autocommit=False,
                  assignment_strategy="org.apache.kafka.clients.consumer.RangeAssignor",
-                 version=DEV_BRANCH, stop_timeout_sec=30, log_level="INFO"):
+                 version=DEV_BRANCH, stop_timeout_sec=30, log_level="INFO", jaas_override_variables=None):
+        """
+        :param jaas_override_variables: A dict of variables to be used in the jaas.conf template file
+        """
         super(VerifiableConsumer, self).__init__(context, num_nodes)
         self.log_level = log_level
         
@@ -178,6 +181,7 @@ def __init__(self, context, num_nodes, kafka, topic, group_id,
         self.event_handlers = {}
         self.global_position = {}
         self.global_committed = {}
+        self.jaas_override_variables = jaas_override_variables or {}
 
         for node in self.nodes:
             node.version = version
@@ -198,7 +202,8 @@ def _worker(self, idx, node):
         node.account.create_file(VerifiableConsumer.LOG4J_CONFIG, log_config)
 
         # Create and upload config file
-        self.security_config = self.kafka.security_config.client_config(self.prop_file, node)
+        self.security_config = self.kafka.security_config.client_config(self.prop_file, node,
+                                                                        self.jaas_override_variables)
         self.security_config.setup_node(node)
         self.prop_file += str(self.security_config)
         self.logger.info("verifiable_consumer.properties:")
diff --git a/tests/kafkatest/services/verifiable_producer.py b/tests/kafkatest/services/verifiable_producer.py
index cbce27e178518..e69504890194d 100644
--- a/tests/kafkatest/services/verifiable_producer.py
+++ b/tests/kafkatest/services/verifiable_producer.py
@@ -57,7 +57,8 @@ class VerifiableProducer(KafkaPathResolverMixin, VerifiableClientMixin, Backgrou
     def __init__(self, context, num_nodes, kafka, topic, max_messages=-1, throughput=100000,
                  message_validator=is_int, compression_types=None, version=DEV_BRANCH, acks=None,
                  stop_timeout_sec=150, request_timeout_sec=30, log_level="INFO",
-                 enable_idempotence=False, offline_nodes=[], create_time=-1, repeating_keys=None):
+                 enable_idempotence=False, offline_nodes=[], create_time=-1, repeating_keys=None,
+                 jaas_override_variables=None):
         """
         :param max_messages is a number of messages to be produced per producer
         :param message_validator checks for an expected format of messages produced. There are
@@ -68,6 +69,7 @@ def __init__(self, context, num_nodes, kafka, topic, max_messages=-1, throughput
                will produce exactly same messages, and validation may miss missing messages.
         :param compression_types: If None, all producers will not use compression; or a list of
         compression types, one per producer (could be "none").
+        :param jaas_override_variables: A dict of variables to be used in the jaas.conf template file
         """
         super(VerifiableProducer, self).__init__(context, num_nodes)
         self.log_level = log_level
@@ -94,6 +96,7 @@ def __init__(self, context, num_nodes, kafka, topic, max_messages=-1, throughput
         self.offline_nodes = offline_nodes
         self.create_time = create_time
         self.repeating_keys = repeating_keys
+        self.jaas_override_variables = jaas_override_variables or {}
 
     def java_class_name(self):
         return "VerifiableProducer"
@@ -116,7 +119,8 @@ def _worker(self, idx, node):
         node.account.create_file(VerifiableProducer.LOG4J_CONFIG, log_config)
 
         # Configure security
-        self.security_config = self.kafka.security_config.client_config(node=node)
+        self.security_config = self.kafka.security_config.client_config(node=node,
+                                                                        jaas_override_variables=self.jaas_override_variables)
         self.security_config.setup_node(node)
 
         # Create and upload config file

From b8559de23d120ca07daa6f66de6bba253d16a74a Mon Sep 17 00:00:00 2001
From: Joan Goyeau 
Date: Fri, 24 Aug 2018 01:07:39 +0100
Subject: [PATCH 0744/1847] MINOR: Fix streams Scala foreach recursive call
 (#5539)

Reviewers: Guozhang Wang , John Roesler 
---
 .../streams/scala/FunctionConversions.scala      |  6 ++++++
 .../kafka/streams/scala/kstream/KStream.scala    | 12 +++++-------
 .../apache/kafka/streams/scala/KStreamTest.scala | 16 ++++++++++++++++
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala
index 65ea490332635..566ba22e615c8 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/FunctionConversions.scala
@@ -34,6 +34,12 @@ import java.lang.{Iterable => JIterable}
  */
 object FunctionConversions {
 
+  implicit private[scala] class ForeachActionFromFunction[K, V](val p: (K, V) => Unit) extends AnyVal {
+    def asForeachAction: ForeachAction[K, V] = new ForeachAction[K, V] {
+      override def apply(key: K, value: V): Unit = p(key, value)
+    }
+  }
+
   implicit class PredicateFromFunction[K, V](val p: (K, V) => Boolean) extends AnyVal {
     def asPredicate: Predicate[K, V] = new Predicate[K, V] {
       override def test(key: K, value: V): Boolean = p(key, value)
diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
index adc1850dc3281..2bcbf0422aefa 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
@@ -22,7 +22,7 @@ package kstream
 
 import org.apache.kafka.streams.KeyValue
 import org.apache.kafka.streams.kstream.{KStream => KStreamJ, _}
-import org.apache.kafka.streams.processor.{Processor, ProcessorContext, ProcessorSupplier, TopicNameExtractor}
+import org.apache.kafka.streams.processor.{Processor, ProcessorSupplier, TopicNameExtractor}
 import org.apache.kafka.streams.scala.ImplicitConversions._
 import org.apache.kafka.streams.scala.FunctionConversions._
 
@@ -84,10 +84,8 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @return a [[KStream]] that contains records with new key and value (possibly both of different type)
    * @see `org.apache.kafka.streams.kstream.KStream#map`
    */
-  def map[KR, VR](mapper: (K, V) => (KR, VR)): KStream[KR, VR] = {
-    val kvMapper = mapper.tupled andThen tuple2ToKeyValue
-    inner.map[KR, VR](((k: K, v: V) => kvMapper(k, v)).asKeyValueMapper)
-  }
+  def map[KR, VR](mapper: (K, V) => (KR, VR)): KStream[KR, VR] =
+    inner.map[KR, VR](mapper.asKeyValueMapper)
 
   /**
    * Transform the value of each input record into a new value (with possible new type) of the output record.
@@ -124,7 +122,7 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @see `org.apache.kafka.streams.kstream.KStream#flatMap`
    */
   def flatMap[KR, VR](mapper: (K, V) => Iterable[(KR, VR)]): KStream[KR, VR] = {
-    val kvMapper = mapper.tupled andThen (iter => iter.map(tuple2ToKeyValue).asJava)
+    val kvMapper = mapper.tupled.andThen(_.map(tuple2ToKeyValue).asJava)
     inner.flatMap[KR, VR](((k: K, v: V) => kvMapper(k, v)).asKeyValueMapper)
   }
 
@@ -173,7 +171,7 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @see `org.apache.kafka.streams.kstream.KStream#foreach`
    */
   def foreach(action: (K, V) => Unit): Unit =
-    inner.foreach((k: K, v: V) => action(k, v))
+    inner.foreach(action.asForeachAction)
 
   /**
    * Creates an array of `KStream` from this stream by branching the records in the original stream based on
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
index 2e2132d14eb89..e70d9007309be 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
@@ -75,6 +75,22 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     testDriver.close()
   }
 
+  "foreach a KStream" should "side effect records" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+
+    var acc = ""
+    builder.stream[String, String](sourceTopic).foreach((_, value) => acc += value)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
+    acc shouldBe "value1value2"
+
+    testDriver.close()
+  }
+
   "selectKey a KStream" should "select a new key" in {
     val builder = new StreamsBuilder()
     val sourceTopic = "source"

From 1dd85a328f437643a5c135f242c644f47c3767e4 Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Thu, 23 Aug 2018 19:53:10 -0500
Subject: [PATCH 0745/1847] MINOR: restructure Windows to favor immutable
 implementation (#5536)

Update to KIP-328.

Reviewers: Matthias J. Sax , Guozhang Wang , Ted Yu , Kamal Chandraprakash 
---
 .../kafka/streams/kstream/JoinWindows.java    |  84 ++++++++--
 .../kafka/streams/kstream/SessionWindows.java |   5 +-
 .../kafka/streams/kstream/TimeWindows.java    |  83 ++++++++--
 .../streams/kstream/UnlimitedWindows.java     |  22 ++-
 .../apache/kafka/streams/kstream/Windows.java |  77 ++-------
 .../kstream/internals/WindowingDefaults.java  |  23 +++
 .../apache/kafka/streams/EqualityCheck.java   | 153 ++++++++++++++++++
 .../KStreamAggregationIntegrationTest.java    |  90 +++++++++++
 .../streams/kstream/JoinWindowsTest.java      |  69 +++-----
 .../streams/kstream/SessionWindowsTest.java   |  34 ++--
 .../streams/kstream/TimeWindowsTest.java      |  52 ++----
 .../streams/kstream/UnlimitedWindowsTest.java |  22 +--
 .../kafka/streams/kstream/WindowsTest.java    |  19 +--
 13 files changed, 481 insertions(+), 252 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowingDefaults.java
 create mode 100644 streams/src/test/java/org/apache/kafka/streams/EqualityCheck.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java
index f91182e8c2f33..5e742e1a25e0d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/JoinWindows.java
@@ -18,9 +18,12 @@
 
 import org.apache.kafka.streams.processor.TimestampExtractor;
 
+import java.time.Duration;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.apache.kafka.streams.kstream.internals.WindowingDefaults.DEFAULT_RETENTION_MS;
+
 /**
  * The window specifications used for joins.
  * 
      
@@ -65,17 +68,39 @@
  */
 public final class JoinWindows extends Windows {
 
+    private final long maintainDurationMs;
+
     /** Maximum time difference for tuples that are before the join tuple. */
     public final long beforeMs;
     /** Maximum time difference for tuples that are after the join tuple. */
     public final long afterMs;
 
-    private JoinWindows(final long beforeMs, final long afterMs) {
+    private final Duration grace;
+
+    private JoinWindows(final long beforeMs, final long afterMs, final Duration grace, final long maintainDurationMs) {
+        if (beforeMs + afterMs < 0) {
+            throw new IllegalArgumentException("Window interval (ie, beforeMs+afterMs) must not be negative.");
+        }
+        this.afterMs = afterMs;
+        this.beforeMs = beforeMs;
+        this.grace = grace;
+        this.maintainDurationMs = maintainDurationMs;
+    }
+
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
+    private JoinWindows(final long beforeMs,
+                        final long afterMs,
+                        final Duration grace,
+                        final long maintainDurationMs,
+                        final int segments) {
+        super(segments);
         if (beforeMs + afterMs < 0) {
             throw new IllegalArgumentException("Window interval (ie, beforeMs+afterMs) must not be negative.");
         }
         this.afterMs = afterMs;
         this.beforeMs = beforeMs;
+        this.grace = grace;
+        this.maintainDurationMs = maintainDurationMs;
     }
 
     /**
@@ -87,7 +112,8 @@ private JoinWindows(final long beforeMs, final long afterMs) {
      * @throws IllegalArgumentException if {@code timeDifferenceMs} is negative
      */
     public static JoinWindows of(final long timeDifferenceMs) throws IllegalArgumentException {
-        return new JoinWindows(timeDifferenceMs, timeDifferenceMs);
+        // This is a static factory method, so we initialize grace and retention to the defaults.
+        return new JoinWindows(timeDifferenceMs, timeDifferenceMs, null, DEFAULT_RETENTION_MS);
     }
 
     /**
@@ -100,8 +126,9 @@ public static JoinWindows of(final long timeDifferenceMs) throws IllegalArgument
      * @param timeDifferenceMs relative window start time in milliseconds
      * @throws IllegalArgumentException if the resulting window size is negative
      */
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
     public JoinWindows before(final long timeDifferenceMs) throws IllegalArgumentException {
-        return new JoinWindows(timeDifferenceMs, afterMs);
+        return new JoinWindows(timeDifferenceMs, afterMs, grace, maintainDurationMs, segments);
     }
 
     /**
@@ -114,8 +141,9 @@ public JoinWindows before(final long timeDifferenceMs) throws IllegalArgumentExc
      * @param timeDifferenceMs relative window end time in milliseconds
      * @throws IllegalArgumentException if the resulting window size is negative
      */
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
     public JoinWindows after(final long timeDifferenceMs) throws IllegalArgumentException {
-        return new JoinWindows(beforeMs, timeDifferenceMs);
+        return new JoinWindows(beforeMs, timeDifferenceMs, grace, maintainDurationMs, segments);
     }
 
     /**
@@ -134,10 +162,30 @@ public long size() {
         return beforeMs + afterMs;
     }
 
-    @Override
+    /**
+     * Reject late events that arrive more than {@code millisAfterWindowEnd}
+     * after the end of its window.
+     *
+     * Lateness is defined as (stream_time - record_timestamp).
+     *
+     * @param millisAfterWindowEnd The grace period to admit late-arriving events to a window.
+     * @return this updated builder
+     */
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
     public JoinWindows grace(final long millisAfterWindowEnd) {
-        super.grace(millisAfterWindowEnd);
-        return this;
+        if (millisAfterWindowEnd < 0) {
+            throw new IllegalArgumentException("Grace period must not be negative.");
+        }
+        return new JoinWindows(beforeMs, afterMs, Duration.ofMillis(millisAfterWindowEnd), maintainDurationMs, segments);
+    }
+
+    @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode
+    @Override
+    public long gracePeriodMs() {
+        // NOTE: in the future, when we remove maintainMs,
+        // we should default the grace period to 24h to maintain the default behavior,
+        // or we can default to (24h - size) if you want to be super accurate.
+        return grace != null ? grace.toMillis() : maintainMs() - size();
     }
 
     /**
@@ -146,14 +194,14 @@ public JoinWindows grace(final long millisAfterWindowEnd) {
      * @throws IllegalArgumentException if {@code durationMs} is smaller than the window size
      * @deprecated since 2.1. Use {@link JoinWindows#grace(long)} instead.
      */
+    @SuppressWarnings("deprecation")
     @Override
     @Deprecated
     public JoinWindows until(final long durationMs) throws IllegalArgumentException {
         if (durationMs < size()) {
             throw new IllegalArgumentException("Window retention time (durationMs) cannot be smaller than the window size.");
         }
-        super.until(durationMs);
-        return this;
+        return new JoinWindows(beforeMs, afterMs, grace, durationMs, segments);
     }
 
     /**
@@ -164,33 +212,41 @@ public JoinWindows until(final long durationMs) throws IllegalArgumentException
      * @return the window maintain duration
      * @deprecated since 2.1. Use {@link JoinWindows#gracePeriodMs()} instead.
      */
+    @SuppressWarnings("deprecation")
     @Override
     @Deprecated
     public long maintainMs() {
-        return Math.max(super.maintainMs(), size());
+        return Math.max(maintainDurationMs, size());
     }
 
+    @SuppressWarnings({"deprecation", "NonFinalFieldReferenceInEquals"}) // removing segments from Windows will fix this
     @Override
     public boolean equals(final Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        if (!super.equals(o)) return false;
         final JoinWindows that = (JoinWindows) o;
         return beforeMs == that.beforeMs &&
-            afterMs == that.afterMs;
+            afterMs == that.afterMs &&
+            maintainDurationMs == that.maintainDurationMs &&
+            segments == that.segments &&
+            Objects.equals(grace, that.grace);
     }
 
+    @SuppressWarnings({"deprecation", "NonFinalFieldReferencedInHashCode"}) // removing segments from Windows will fix this
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), beforeMs, afterMs);
+        return Objects.hash(beforeMs, afterMs, grace, maintainDurationMs, segments);
     }
 
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
     @Override
     public String toString() {
         return "JoinWindows{" +
             "beforeMs=" + beforeMs +
             ", afterMs=" + afterMs +
-            ", super=" + super.toString() +
+            ", grace=" + grace +
+            ", maintainDurationMs=" + maintainDurationMs +
+            ", segments=" + segments +
             '}';
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
index c4dd91d3ed506..9efb78c786650 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
@@ -22,6 +22,8 @@
 import java.time.Duration;
 import java.util.Objects;
 
+import static org.apache.kafka.streams.kstream.internals.WindowingDefaults.DEFAULT_RETENTION_MS;
+
 
 /**
  * A session based window specification used for aggregating events into sessions.
@@ -91,8 +93,7 @@ public static SessionWindows with(final long inactivityGapMs) {
         if (inactivityGapMs <= 0) {
             throw new IllegalArgumentException("Gap time (inactivityGapMs) cannot be zero or negative.");
         }
-        final long oneDayMs = 24 * 60 * 60_000L;
-        return new SessionWindows(inactivityGapMs, oneDayMs, null);
+        return new SessionWindows(inactivityGapMs, DEFAULT_RETENTION_MS, null);
     }
 
     /**
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java
index 8b62a43a8b3b4..6a58c2c09c4d7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java
@@ -20,10 +20,13 @@
 import org.apache.kafka.streams.processor.TimestampExtractor;
 import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
 
+import java.time.Duration;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.apache.kafka.streams.kstream.internals.WindowingDefaults.DEFAULT_RETENTION_MS;
+
 /**
  * The fixed-size time-based window specifications used for aggregations.
  * 
      
@@ -52,6 +55,8 @@
  */
 public final class TimeWindows extends Windows {
 
+    private final long maintainDurationMs;
+
     /** The size of the windows in milliseconds. */
     public final long sizeMs;
 
@@ -60,10 +65,28 @@ public final class TimeWindows extends Windows {
      * the previous one.
      */
     public final long advanceMs;
+    private final Duration grace;
+
+    private TimeWindows(final long sizeMs, final long advanceMs, final Duration grace, final long maintainDurationMs) {
+        this.sizeMs = sizeMs;
+        this.advanceMs = advanceMs;
+        this.grace = grace;
+        this.maintainDurationMs = maintainDurationMs;
+    }
 
-    private TimeWindows(final long sizeMs, final long advanceMs) {
+    /** Private constructor for preserving segments. Can be removed along with Windows.segments. **/
+    @SuppressWarnings("DeprecatedIsStillUsed")
+    @Deprecated
+    private TimeWindows(final long sizeMs,
+                        final long advanceMs,
+                        final Duration grace,
+                        final long maintainDurationMs,
+                        final int segments) {
+        super(segments);
         this.sizeMs = sizeMs;
         this.advanceMs = advanceMs;
+        this.grace = grace;
+        this.maintainDurationMs = maintainDurationMs;
     }
 
     /**
@@ -82,7 +105,8 @@ public static TimeWindows of(final long sizeMs) throws IllegalArgumentException
         if (sizeMs <= 0) {
             throw new IllegalArgumentException("Window size (sizeMs) must be larger than zero.");
         }
-        return new TimeWindows(sizeMs, sizeMs);
+        // This is a static factory method, so we initialize grace and retention to the defaults.
+        return new TimeWindows(sizeMs, sizeMs, null, DEFAULT_RETENTION_MS);
     }
 
     /**
@@ -97,11 +121,12 @@ public static TimeWindows of(final long sizeMs) throws IllegalArgumentException
      * @return a new window definition with default maintain duration of 1 day
      * @throws IllegalArgumentException if the advance interval is negative, zero, or larger-or-equal the window size
      */
+    @SuppressWarnings("deprecation") // will be fixed when we remove segments from Windows
     public TimeWindows advanceBy(final long advanceMs) {
         if (advanceMs <= 0 || advanceMs > sizeMs) {
             throw new IllegalArgumentException(String.format("AdvanceMs must lie within interval (0, %d].", sizeMs));
         }
-        return new TimeWindows(sizeMs, advanceMs);
+        return new TimeWindows(sizeMs, advanceMs, grace, maintainDurationMs, segments);
     }
 
     @Override
@@ -121,10 +146,30 @@ public long size() {
         return sizeMs;
     }
 
-    @Override
+    /**
+     * Reject late events that arrive more than {@code millisAfterWindowEnd}
+     * after the end of its window.
+     *
+     * Lateness is defined as (stream_time - record_timestamp).
+     *
+     * @param millisAfterWindowEnd The grace period to admit late-arriving events to a window.
+     * @return this updated builder
+     */
+    @SuppressWarnings("deprecation") // will be fixed when we remove segments from Windows
     public TimeWindows grace(final long millisAfterWindowEnd) {
-        super.grace(millisAfterWindowEnd);
-        return this;
+        if (millisAfterWindowEnd < 0) {
+            throw new IllegalArgumentException("Grace period must not be negative.");
+        }
+        return new TimeWindows(sizeMs, advanceMs, Duration.ofMillis(millisAfterWindowEnd), maintainDurationMs, segments);
+    }
+
+    @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode
+    @Override
+    public long gracePeriodMs() {
+        // NOTE: in the future, when we remove maintainMs,
+        // we should default the grace period to 24h to maintain the default behavior,
+        // or we can default to (24h - size) if you want to be super accurate.
+        return grace != null ? grace.toMillis() : maintainMs() - size();
     }
 
     /**
@@ -135,14 +180,14 @@ public TimeWindows grace(final long millisAfterWindowEnd) {
      * @deprecated since 2.1. Use {@link Materialized#retention} or directly configure the retention in a store supplier
      *             and use {@link Materialized#as(WindowBytesStoreSupplier)}.
      */
+    @SuppressWarnings("deprecation")
     @Override
     @Deprecated
     public TimeWindows until(final long durationMs) throws IllegalArgumentException {
         if (durationMs < sizeMs) {
             throw new IllegalArgumentException("Window retention time (durationMs) cannot be smaller than the window size.");
         }
-        super.until(durationMs);
-        return this;
+        return new TimeWindows(sizeMs, advanceMs, grace, durationMs, segments);
     }
 
     /**
@@ -153,33 +198,41 @@ public TimeWindows until(final long durationMs) throws IllegalArgumentException
      * @return the window maintain duration
      * @deprecated since 2.1. Use {@link Materialized#retention} instead.
      */
+    @SuppressWarnings({"DeprecatedIsStillUsed", "deprecation"})
     @Override
     @Deprecated
     public long maintainMs() {
-        return Math.max(super.maintainMs(), sizeMs);
+        return Math.max(maintainDurationMs, sizeMs);
     }
 
+    @SuppressWarnings({"deprecation", "NonFinalFieldReferenceInEquals"}) // removing segments from Windows will fix this
     @Override
     public boolean equals(final Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        if (!super.equals(o)) return false;
         final TimeWindows that = (TimeWindows) o;
-        return sizeMs == that.sizeMs &&
-            advanceMs == that.advanceMs;
+        return maintainDurationMs == that.maintainDurationMs &&
+            segments == that.segments &&
+            sizeMs == that.sizeMs &&
+            advanceMs == that.advanceMs &&
+            Objects.equals(grace, that.grace);
     }
 
+    @SuppressWarnings({"deprecation", "NonFinalFieldReferencedInHashCode"}) // removing segments from Windows will fix this
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), sizeMs, advanceMs);
+        return Objects.hash(maintainDurationMs, segments, sizeMs, advanceMs, grace);
     }
 
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
     @Override
     public String toString() {
         return "TimeWindows{" +
-            "sizeMs=" + sizeMs +
+            "maintainDurationMs=" + maintainDurationMs +
+            ", sizeMs=" + sizeMs +
             ", advanceMs=" + advanceMs +
-            ", super=" + super.toString() +
+            ", grace=" + grace +
+            ", segments=" + segments +
             '}';
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java
index cf4482fb27ba2..73aa9b1cd2abe 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/UnlimitedWindows.java
@@ -100,6 +100,7 @@ public long size() {
      * @throws IllegalArgumentException on every invocation.
      * @deprecated since 2.1.
      */
+    @SuppressWarnings("deprecation")
     @Override
     @Deprecated
     public UnlimitedWindows until(final long durationMs) {
@@ -113,42 +114,39 @@ public UnlimitedWindows until(final long durationMs) {
      * @return the window retention time that is {@link Long#MAX_VALUE}
      * @deprecated since 2.1. Use {@link Materialized#retention} instead.
      */
+    @SuppressWarnings("deprecation")
     @Override
     @Deprecated
     public long maintainMs() {
         return Long.MAX_VALUE;
     }
 
-    /**
-     * Throws an {@link IllegalArgumentException} because the window never ends and the
-     * grace period is therefore meaningless.
-     *
-     * @throws IllegalArgumentException on every invocation
-     */
     @Override
-    public UnlimitedWindows grace(final long millisAfterWindowEnd) {
-        throw new IllegalArgumentException("Grace period cannot be set for UnlimitedWindows.");
+    public long gracePeriodMs() {
+        return 0L;
     }
 
+    @SuppressWarnings({"deprecation", "NonFinalFieldReferenceInEquals"}) // removing segments from Windows will fix this
     @Override
     public boolean equals(final Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        if (!super.equals(o)) return false;
         final UnlimitedWindows that = (UnlimitedWindows) o;
-        return startMs == that.startMs;
+        return startMs == that.startMs && segments == that.segments;
     }
 
+    @SuppressWarnings({"deprecation", "NonFinalFieldReferencedInHashCode"}) // removing segments from Windows will fix this
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), startMs);
+        return Objects.hash(startMs, segments);
     }
 
+    @SuppressWarnings({"deprecation"}) // removing segments from Windows will fix this
     @Override
     public String toString() {
         return "UnlimitedWindows{" +
             "startMs=" + startMs +
-            ", super=" + super.toString() +
+            ", segments=" + segments +
             '}';
     }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
index 8c9ba55f88daf..b6b21785bbee9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
@@ -19,9 +19,9 @@
 import org.apache.kafka.streams.processor.TimestampExtractor;
 import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
 
-import java.time.Duration;
 import java.util.Map;
-import java.util.Objects;
+
+import static org.apache.kafka.streams.kstream.internals.WindowingDefaults.DEFAULT_RETENTION_MS;
 
 /**
  * The window specification for fixed size windows that is used to define window boundaries and grace period.
@@ -40,44 +40,14 @@
  */
 public abstract class Windows {
 
-    private long maintainDurationMs = 24 * 60 * 60 * 1000L; // default: one day
+    private long maintainDurationMs = DEFAULT_RETENTION_MS;
     @Deprecated public int segments = 3;
 
-    private Duration grace;
-
     protected Windows() {}
 
-    /**
-     * Reject late events that arrive more than {@code millisAfterWindowEnd}
-     * after the end of its window.
-     *
-     * Lateness is defined as (stream_time - record_timestamp).
-     *
-     * @param millisAfterWindowEnd The grace period to admit late-arriving events to a window.
-     * @return this updated builder
-     */
-    public Windows grace(final long millisAfterWindowEnd) {
-        if (millisAfterWindowEnd < 0) {
-            throw new IllegalArgumentException("Grace period must not be negative.");
-        }
-
-        grace = Duration.ofMillis(millisAfterWindowEnd);
-
-        return this;
-    }
-
-    /**
-     * Return the window grace period (the time to admit
-     * late-arriving events after the end of the window.)
-     *
-     * Lateness is defined as (stream_time - record_timestamp).
-     */
-    @SuppressWarnings("deprecation") // continuing to support Windows#maintainMs/segmentInterval in fallback mode
-    public long gracePeriodMs() {
-        // NOTE: in the future, when we remove maintainMs,
-        // we should default the grace period to 24h to maintain the default behavior,
-        // or we can default to (24h - size) if you want to be super accurate.
-        return grace != null ? grace.toMillis() : maintainMs() - size();
+    @SuppressWarnings("deprecation") // remove this constructor when we remove segments.
+    Windows(final int segments) {
+        this.segments = segments;
     }
 
     /**
@@ -106,6 +76,7 @@ public Windows until(final long durationMs) throws IllegalArgumentException {
      * @return the window maintain duration
      * @deprecated since 2.1. Use {@link Materialized#retention} instead.
      */
+    @SuppressWarnings("DeprecatedIsStillUsed")
     @Deprecated
     public long maintainMs() {
         return maintainDurationMs;
@@ -161,34 +132,10 @@ protected Windows segments(final int segments) throws IllegalArgumentExceptio
     public abstract long size();
 
     /**
-     * Warning: It may be unsafe to use objects of this class in set- or map-like collections,
-     * since the equals and hashCode methods depend on mutable fields.
-     */
-    @Override
-    public boolean equals(final Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        final Windows windows = (Windows) o;
-        return maintainMs() == windows.maintainMs() &&
-            segments == windows.segments &&
-            Objects.equals(gracePeriodMs(), windows.gracePeriodMs());
-    }
-
-    /**
-     * Warning: It may be unsafe to use objects of this class in set- or map-like collections,
-     * since the equals and hashCode methods depend on mutable fields.
+     * Return the window grace period (the time to admit
+     * late-arriving events after the end of the window.)
+     *
+     * Lateness is defined as (stream_time - record_timestamp).
      */
-    @Override
-    public int hashCode() {
-        return Objects.hash(maintainMs(), segments, gracePeriodMs());
-    }
-
-    @Override
-    public String toString() {
-        return "Windows{" +
-            "maintainDurationMs=" + maintainMs() +
-            ", segments=" + segments +
-            ", grace=" + gracePeriodMs() +
-            '}';
-    }
+    public abstract long gracePeriodMs();
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowingDefaults.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowingDefaults.java
new file mode 100644
index 0000000000000..50015c1b56115
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowingDefaults.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals;
+
+public final class WindowingDefaults {
+    private WindowingDefaults() {}
+
+    public static final long DEFAULT_RETENTION_MS = 24 * 60 * 60 * 1000L; // one day
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/EqualityCheck.java b/streams/src/test/java/org/apache/kafka/streams/EqualityCheck.java
new file mode 100644
index 0000000000000..4ca1dde8b2068
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/EqualityCheck.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams;
+
+public final class EqualityCheck {
+    private EqualityCheck() {}
+
+    // Inspired by EqualsTester from Guava
+    public static  void verifyEquality(final T o1, final T o2) {
+        // making sure we don't get an NPE in the test
+        if (o1 == null && o2 == null) {
+            return;
+        } else if (o1 == null) {
+            throw new AssertionError(String.format("o1 was null, but o2[%s] was not.", o2));
+        } else if (o2 == null) {
+            throw new AssertionError(String.format("o1[%s] was not null, but o2 was.", o1));
+        }
+        verifyGeneralEqualityProperties(o1, o2);
+
+
+        // the two objects should equal each other
+        if (!o1.equals(o2)) {
+            throw new AssertionError(String.format("o1[%s] was not equal to o2[%s].", o1, o2));
+        }
+
+        if (!o2.equals(o1)) {
+            throw new AssertionError(String.format("o2[%s] was not equal to o1[%s].", o2, o1));
+        }
+
+        verifyHashCodeConsistency(o1, o2);
+
+        // since these objects are equal, their hashcode MUST be the same
+        if (o1.hashCode() != o2.hashCode()) {
+            throw new AssertionError(String.format("o1[%s].hash[%d] was not equal to o2[%s].hash[%d].", o1, o1.hashCode(), o2, o2.hashCode()));
+        }
+    }
+
+    public static  void verifyInEquality(final T o1, final T o2) {
+        // making sure we don't get an NPE in the test
+        if (o1 == null && o2 == null) {
+            throw new AssertionError("Both o1 and o2 were null.");
+        } else if (o1 == null) {
+            return;
+        } else if (o2 == null) {
+            return;
+        }
+
+        verifyGeneralEqualityProperties(o1, o2);
+
+        // these two objects should NOT equal each other
+        if (o1.equals(o2)) {
+            throw new AssertionError(String.format("o1[%s] was equal to o2[%s].", o1, o2));
+        }
+
+        if (o2.equals(o1)) {
+            throw new AssertionError(String.format("o2[%s] was equal to o1[%s].", o2, o1));
+        }
+        verifyHashCodeConsistency(o1, o2);
+
+
+        // since these objects are NOT equal, their hashcode SHOULD PROBABLY not be the same
+        if (o1.hashCode() == o2.hashCode()) {
+            throw new AssertionError(
+                String.format(
+                    "o1[%s].hash[%d] was equal to o2[%s].hash[%d], even though !o1.equals(o2). " +
+                        "This is NOT A BUG, but it is undesirable for hash collection performance.",
+                    o1,
+                    o1.hashCode(),
+                    o2,
+                    o2.hashCode()
+                )
+            );
+        }
+    }
+
+
+    @SuppressWarnings({"EqualsWithItself", "ConstantConditions", "ObjectEqualsNull"})
+    private static  void verifyGeneralEqualityProperties(final T o1, final T o2) {
+        // objects should equal themselves
+        if (!o1.equals(o1)) {
+            throw new AssertionError(String.format("o1[%s] was not equal to itself.", o1));
+        }
+
+        if (!o2.equals(o2)) {
+            throw new AssertionError(String.format("o2[%s] was not equal to itself.", o2));
+        }
+
+        // non-null objects should not equal null
+        if (o1.equals(null)) {
+            throw new AssertionError(String.format("o1[%s] was equal to null.", o1));
+        }
+
+        if (o2.equals(null)) {
+            throw new AssertionError(String.format("o2[%s] was equal to null.", o2));
+        }
+
+        // objects should not equal some random object
+        if (o1.equals(new Object())) {
+            throw new AssertionError(String.format("o1[%s] was equal to an anonymous Object.", o1));
+        }
+
+        if (o2.equals(new Object())) {
+            throw new AssertionError(String.format("o2[%s] was equal to an anonymous Object.", o2));
+        }
+    }
+
+
+    private static  void verifyHashCodeConsistency(final T o1, final T o2) {
+        {
+            final int first = o1.hashCode();
+            final int second = o1.hashCode();
+            if (first != second) {
+                throw new AssertionError(
+                    String.format(
+                        "o1[%s]'s hashcode was not consistent: [%d]!=[%d].",
+                        o1,
+                        first,
+                        second
+                    )
+                );
+            }
+        }
+
+        {
+            final int first = o2.hashCode();
+            final int second = o2.hashCode();
+            if (first != second) {
+                throw new AssertionError(
+                    String.format(
+                        "o2[%s]'s hashcode was not consistent: [%d]!=[%d].",
+                        o2,
+                        first,
+                        second
+                    )
+                );
+            }
+        }
+    }
+}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
index c16950e02d744..ce6c352af9c38 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
@@ -49,10 +49,12 @@
 import org.apache.kafka.streams.kstream.TimeWindowedDeserializer;
 import org.apache.kafka.streams.kstream.TimeWindows;
 import org.apache.kafka.streams.kstream.Transformer;
+import org.apache.kafka.streams.kstream.UnlimitedWindows;
 import org.apache.kafka.streams.kstream.Windowed;
 import org.apache.kafka.streams.kstream.WindowedSerdes;
 import org.apache.kafka.streams.kstream.internals.SessionWindow;
 import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.kstream.internals.UnlimitedWindow;
 import org.apache.kafka.streams.processor.ProcessorContext;
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.KeyValueStore;
@@ -70,6 +72,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
@@ -645,6 +648,93 @@ public void shouldReduceSessionWindows() throws Exception {
 
     }
 
+    @Test
+    public void shouldCountUnlimitedWindows() throws Exception {
+        final long startTime = mockTime.milliseconds() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS) + 1;
+        final long incrementTime = Duration.ofDays(1).toMillis();
+
+        final long t1 = mockTime.milliseconds() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
+        final List> t1Messages = Arrays.asList(new KeyValue<>("bob", "start"),
+                                                                        new KeyValue<>("penny", "start"),
+                                                                        new KeyValue<>("jo", "pause"),
+                                                                        new KeyValue<>("emily", "pause"));
+
+        final Properties producerConfig = TestUtils.producerConfig(
+            CLUSTER.bootstrapServers(),
+            StringSerializer.class,
+            StringSerializer.class,
+            new Properties()
+        );
+
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            userSessionsStream,
+            t1Messages,
+            producerConfig,
+            t1);
+
+        final long t2 = t1 + incrementTime;
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            userSessionsStream,
+            Collections.singletonList(
+                new KeyValue<>("emily", "resume")
+            ),
+            producerConfig,
+            t2);
+        final long t3 = t2 + incrementTime;
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            userSessionsStream,
+            Arrays.asList(
+                new KeyValue<>("bob", "pause"),
+                new KeyValue<>("penny", "stop")
+            ),
+            producerConfig,
+            t3);
+
+        final long t4 = t3 + incrementTime;
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+            userSessionsStream,
+            Arrays.asList(
+                new KeyValue<>("bob", "resume"), // bobs session continues
+                new KeyValue<>("jo", "resume")   // jo's starts new session
+            ),
+            producerConfig,
+            t4);
+
+        final Map, KeyValue> results = new HashMap<>();
+        final CountDownLatch latch = new CountDownLatch(5);
+
+        builder.stream(userSessionsStream, Consumed.with(Serdes.String(), Serdes.String()))
+               .groupByKey(Serialized.with(Serdes.String(), Serdes.String()))
+               .windowedBy(UnlimitedWindows.of().startOn(startTime))
+               .count()
+               .toStream()
+               .transform(() -> new Transformer, Long, KeyValue>() {
+                   private ProcessorContext context;
+
+                   @Override
+                   public void init(final ProcessorContext context) {
+                       this.context = context;
+                   }
+
+                   @Override
+                   public KeyValue transform(final Windowed key, final Long value) {
+                       results.put(key, KeyValue.pair(value, context.timestamp()));
+                       latch.countDown();
+                       return null;
+                   }
+
+                   @Override
+                   public void close() {}
+               });
+
+        startStreams();
+        assertTrue(latch.await(30, TimeUnit.SECONDS));
+        assertThat(results.get(new Windowed<>("bob", new UnlimitedWindow(startTime))), equalTo(KeyValue.pair(2L, t4)));
+        assertThat(results.get(new Windowed<>("penny", new UnlimitedWindow(startTime))), equalTo(KeyValue.pair(1L, t3)));
+        assertThat(results.get(new Windowed<>("jo", new UnlimitedWindow(startTime))), equalTo(KeyValue.pair(1L, t4)));
+        assertThat(results.get(new Windowed<>("emily", new UnlimitedWindow(startTime))), equalTo(KeyValue.pair(1L, t2)));
+    }
+
 
     private void produceMessages(final long timestamp) throws Exception {
         IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/JoinWindowsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/JoinWindowsTest.java
index ae7937125b70a..de635b441f428 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/JoinWindowsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/JoinWindowsTest.java
@@ -18,8 +18,9 @@
 
 import org.junit.Test;
 
+import static org.apache.kafka.streams.EqualityCheck.verifyEquality;
+import static org.apache.kafka.streams.EqualityCheck.verifyInEquality;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.fail;
 
 
@@ -105,92 +106,58 @@ public void gracePeriodShouldEnforceBoundaries() {
 
     @Test
     public void equalsAndHashcodeShouldBeValidForPositiveCases() {
-        assertEquals(JoinWindows.of(3), JoinWindows.of(3));
-        assertEquals(JoinWindows.of(3).hashCode(), JoinWindows.of(3).hashCode());
+        verifyEquality(JoinWindows.of(3), JoinWindows.of(3));
 
-        assertEquals(JoinWindows.of(3).after(2), JoinWindows.of(3).after(2));
-        assertEquals(JoinWindows.of(3).after(2).hashCode(), JoinWindows.of(3).after(2).hashCode());
+        verifyEquality(JoinWindows.of(3).after(2), JoinWindows.of(3).after(2));
 
-        assertEquals(JoinWindows.of(3).before(2), JoinWindows.of(3).before(2));
-        assertEquals(JoinWindows.of(3).before(2).hashCode(), JoinWindows.of(3).before(2).hashCode());
+        verifyEquality(JoinWindows.of(3).before(2), JoinWindows.of(3).before(2));
 
-        assertEquals(JoinWindows.of(3).grace(2), JoinWindows.of(3).grace(2));
-        assertEquals(JoinWindows.of(3).grace(2).hashCode(), JoinWindows.of(3).grace(2).hashCode());
+        verifyEquality(JoinWindows.of(3).grace(2), JoinWindows.of(3).grace(2));
 
-        assertEquals(JoinWindows.of(3).until(60), JoinWindows.of(3).until(60));
-        assertEquals(JoinWindows.of(3).until(60).hashCode(), JoinWindows.of(3).until(60).hashCode());
+        verifyEquality(JoinWindows.of(3).until(60), JoinWindows.of(3).until(60));
 
-        assertEquals(
+        verifyEquality(
             JoinWindows.of(3).before(1).after(2).grace(3).until(60),
             JoinWindows.of(3).before(1).after(2).grace(3).until(60)
         );
-        assertEquals(
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode(),
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode()
-        );
         // JoinWindows is a little weird in that before and after set the same fields as of.
-        assertEquals(
+        verifyEquality(
             JoinWindows.of(9).before(1).after(2).grace(3).until(60),
             JoinWindows.of(3).before(1).after(2).grace(3).until(60)
         );
-        assertEquals(
-            JoinWindows.of(9).before(1).after(2).grace(3).until(60).hashCode(),
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode()
-        );
     }
 
     @Test
     public void equalsAndHashcodeShouldBeValidForNegativeCases() {
-        assertNotEquals(JoinWindows.of(9), JoinWindows.of(3));
-        assertNotEquals(JoinWindows.of(9).hashCode(), JoinWindows.of(3).hashCode());
+        verifyInEquality(JoinWindows.of(9), JoinWindows.of(3));
 
-        assertNotEquals(JoinWindows.of(3).after(9), JoinWindows.of(3).after(2));
-        assertNotEquals(JoinWindows.of(3).after(9).hashCode(), JoinWindows.of(3).after(2).hashCode());
+        verifyInEquality(JoinWindows.of(3).after(9), JoinWindows.of(3).after(2));
 
-        assertNotEquals(JoinWindows.of(3).before(9), JoinWindows.of(3).before(2));
-        assertNotEquals(JoinWindows.of(3).before(9).hashCode(), JoinWindows.of(3).before(2).hashCode());
+        verifyInEquality(JoinWindows.of(3).before(9), JoinWindows.of(3).before(2));
 
-        assertNotEquals(JoinWindows.of(3).grace(9), JoinWindows.of(3).grace(2));
-        assertNotEquals(JoinWindows.of(3).grace(9).hashCode(), JoinWindows.of(3).grace(2).hashCode());
+        verifyInEquality(JoinWindows.of(3).grace(9), JoinWindows.of(3).grace(2));
 
-        assertNotEquals(JoinWindows.of(3).until(90), JoinWindows.of(3).until(60));
-        assertNotEquals(JoinWindows.of(3).until(90).hashCode(), JoinWindows.of(3).until(60).hashCode());
+        verifyInEquality(JoinWindows.of(3).until(90), JoinWindows.of(3).until(60));
 
 
-        assertNotEquals(
+        verifyInEquality(
             JoinWindows.of(3).before(9).after(2).grace(3).until(60),
             JoinWindows.of(3).before(1).after(2).grace(3).until(60)
         );
-        assertNotEquals(
-            JoinWindows.of(3).before(9).after(2).grace(3).until(60).hashCode(),
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode()
-        );
 
-        assertNotEquals(
+        verifyInEquality(
             JoinWindows.of(3).before(1).after(9).grace(3).until(60),
             JoinWindows.of(3).before(1).after(2).grace(3).until(60)
         );
-        assertNotEquals(
-            JoinWindows.of(3).before(1).after(9).grace(3).until(60).hashCode(),
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode()
-        );
 
-        assertNotEquals(
+        verifyInEquality(
             JoinWindows.of(3).before(1).after(2).grace(9).until(60),
             JoinWindows.of(3).before(1).after(2).grace(3).until(60)
         );
-        assertNotEquals(
-            JoinWindows.of(3).before(1).after(2).grace(9).until(60).hashCode(),
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode()
-        );
 
-        assertNotEquals(
+        verifyInEquality(
             JoinWindows.of(3).before(1).after(2).grace(3).until(90),
             JoinWindows.of(3).before(1).after(2).grace(3).until(60)
         );
-        assertNotEquals(
-            JoinWindows.of(3).before(1).after(2).grace(3).until(90).hashCode(),
-            JoinWindows.of(3).before(1).after(2).grace(3).until(60).hashCode()
-        );
     }
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/SessionWindowsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/SessionWindowsTest.java
index 2acd5d2ba6424..9f99be45cbe8e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/SessionWindowsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/SessionWindowsTest.java
@@ -18,8 +18,9 @@
 
 import org.junit.Test;
 
+import static org.apache.kafka.streams.EqualityCheck.verifyEquality;
+import static org.apache.kafka.streams.EqualityCheck.verifyInEquality;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.fail;
 
 public class SessionWindowsTest {
@@ -81,38 +82,27 @@ public void retentionTimeMustNotBeNegative() {
 
     @Test
     public void equalsAndHashcodeShouldBeValidForPositiveCases() {
-        assertEquals(SessionWindows.with(1), SessionWindows.with(1));
-        assertEquals(SessionWindows.with(1).hashCode(), SessionWindows.with(1).hashCode());
+        verifyEquality(SessionWindows.with(1), SessionWindows.with(1));
 
-        assertEquals(SessionWindows.with(1).grace(6), SessionWindows.with(1).grace(6));
-        assertEquals(SessionWindows.with(1).grace(6).hashCode(), SessionWindows.with(1).grace(6).hashCode());
+        verifyEquality(SessionWindows.with(1).grace(6), SessionWindows.with(1).grace(6));
 
-        assertEquals(SessionWindows.with(1).until(7), SessionWindows.with(1).until(7));
-        assertEquals(SessionWindows.with(1).until(7).hashCode(), SessionWindows.with(1).until(7).hashCode());
+        verifyEquality(SessionWindows.with(1).until(7), SessionWindows.with(1).until(7));
 
-        assertEquals(SessionWindows.with(1).grace(6).until(7), SessionWindows.with(1).grace(6).until(7));
-        assertEquals(SessionWindows.with(1).grace(6).until(7).hashCode(), SessionWindows.with(1).grace(6).until(7).hashCode());
+        verifyEquality(SessionWindows.with(1).grace(6).until(7), SessionWindows.with(1).grace(6).until(7));
     }
 
     @Test
     public void equalsAndHashcodeShouldBeValidForNegativeCases() {
-        assertNotEquals(SessionWindows.with(9), SessionWindows.with(1));
-        assertNotEquals(SessionWindows.with(9).hashCode(), SessionWindows.with(1).hashCode());
+        verifyInEquality(SessionWindows.with(9), SessionWindows.with(1));
 
-        assertNotEquals(SessionWindows.with(1).grace(9), SessionWindows.with(1).grace(6));
-        assertNotEquals(SessionWindows.with(1).grace(9).hashCode(), SessionWindows.with(1).grace(6).hashCode());
+        verifyInEquality(SessionWindows.with(1).grace(9), SessionWindows.with(1).grace(6));
 
-        assertNotEquals(SessionWindows.with(1).until(9), SessionWindows.with(1).until(7));
-        assertNotEquals(SessionWindows.with(1).until(9).hashCode(), SessionWindows.with(1).until(7).hashCode());
+        verifyInEquality(SessionWindows.with(1).until(9), SessionWindows.with(1).until(7));
 
+        verifyInEquality(SessionWindows.with(2).grace(6).until(7), SessionWindows.with(1).grace(6).until(7));
 
-        assertNotEquals(SessionWindows.with(2).grace(6).until(7), SessionWindows.with(1).grace(6).until(7));
-        assertNotEquals(SessionWindows.with(2).grace(6).until(7).hashCode(), SessionWindows.with(1).grace(6).until(7).hashCode());
+        verifyInEquality(SessionWindows.with(1).grace(0).until(7), SessionWindows.with(1).grace(6).until(7));
 
-        assertNotEquals(SessionWindows.with(1).grace(0).until(7), SessionWindows.with(1).grace(6).until(7));
-        assertNotEquals(SessionWindows.with(1).grace(0).until(7).hashCode(), SessionWindows.with(1).grace(6).until(7).hashCode());
-
-        assertNotEquals(SessionWindows.with(1).grace(6).until(70), SessionWindows.with(1).grace(6).until(7));
-        assertNotEquals(SessionWindows.with(1).grace(6).until(70).hashCode(), SessionWindows.with(1).grace(6).until(7).hashCode());
+        verifyInEquality(SessionWindows.with(1).grace(6).until(70), SessionWindows.with(1).grace(6).until(7));
     }
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java
index d426b2811b7f9..9010bb2c27754 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java
@@ -21,6 +21,8 @@
 
 import java.util.Map;
 
+import static org.apache.kafka.streams.EqualityCheck.verifyEquality;
+import static org.apache.kafka.streams.EqualityCheck.verifyInEquality;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.fail;
@@ -150,77 +152,49 @@ public void shouldComputeWindowsForTumblingWindows() {
 
     @Test
     public void equalsAndHashcodeShouldBeValidForPositiveCases() {
-        assertEquals(TimeWindows.of(3), TimeWindows.of(3));
-        assertEquals(TimeWindows.of(3).hashCode(), TimeWindows.of(3).hashCode());
+        verifyEquality(TimeWindows.of(3), TimeWindows.of(3));
 
-        assertEquals(TimeWindows.of(3).advanceBy(1), TimeWindows.of(3).advanceBy(1));
-        assertEquals(TimeWindows.of(3).advanceBy(1).hashCode(), TimeWindows.of(3).advanceBy(1).hashCode());
+        verifyEquality(TimeWindows.of(3).advanceBy(1), TimeWindows.of(3).advanceBy(1));
 
-        assertEquals(TimeWindows.of(3).grace(1), TimeWindows.of(3).grace(1));
-        assertEquals(TimeWindows.of(3).grace(1).hashCode(), TimeWindows.of(3).grace(1).hashCode());
+        verifyEquality(TimeWindows.of(3).grace(1), TimeWindows.of(3).grace(1));
 
-        assertEquals(TimeWindows.of(3).until(4), TimeWindows.of(3).until(4));
-        assertEquals(TimeWindows.of(3).until(4).hashCode(), TimeWindows.of(3).until(4).hashCode());
+        verifyEquality(TimeWindows.of(3).until(4), TimeWindows.of(3).until(4));
 
-        assertEquals(
+        verifyEquality(
             TimeWindows.of(3).advanceBy(1).grace(1).until(4),
             TimeWindows.of(3).advanceBy(1).grace(1).until(4)
         );
-        assertEquals(
-            TimeWindows.of(3).advanceBy(1).grace(1).until(4).hashCode(),
-            TimeWindows.of(3).advanceBy(1).grace(1).until(4).hashCode()
-        );
     }
 
     @Test
     public void equalsAndHashcodeShouldBeValidForNegativeCases() {
-        assertNotEquals(TimeWindows.of(9), TimeWindows.of(3));
-        assertNotEquals(TimeWindows.of(9).hashCode(), TimeWindows.of(3).hashCode());
+        verifyInEquality(TimeWindows.of(9), TimeWindows.of(3));
 
-        assertNotEquals(TimeWindows.of(3).advanceBy(2), TimeWindows.of(3).advanceBy(1));
-        assertNotEquals(TimeWindows.of(3).advanceBy(2).hashCode(), TimeWindows.of(3).advanceBy(1).hashCode());
+        verifyInEquality(TimeWindows.of(3).advanceBy(2), TimeWindows.of(3).advanceBy(1));
 
-        assertNotEquals(TimeWindows.of(3).grace(2), TimeWindows.of(3).grace(1));
-        assertNotEquals(TimeWindows.of(3).grace(2).hashCode(), TimeWindows.of(3).grace(1).hashCode());
+        verifyInEquality(TimeWindows.of(3).grace(2), TimeWindows.of(3).grace(1));
 
-        assertNotEquals(TimeWindows.of(3).until(9), TimeWindows.of(3).until(4));
-        assertNotEquals(TimeWindows.of(3).until(9).hashCode(), TimeWindows.of(3).until(4).hashCode());
+        verifyInEquality(TimeWindows.of(3).until(9), TimeWindows.of(3).until(4));
 
 
-        assertNotEquals(
+        verifyInEquality(
             TimeWindows.of(4).advanceBy(2).grace(2).until(4),
             TimeWindows.of(3).advanceBy(2).grace(2).until(4)
         );
-        assertNotEquals(
-            TimeWindows.of(4).advanceBy(2).grace(2).until(4).hashCode(),
-            TimeWindows.of(3).advanceBy(2).grace(2).until(4).hashCode()
-        );
 
-        assertNotEquals(
+        verifyInEquality(
             TimeWindows.of(3).advanceBy(1).grace(2).until(4),
             TimeWindows.of(3).advanceBy(2).grace(2).until(4)
         );
-        assertNotEquals(
-            TimeWindows.of(3).advanceBy(1).grace(2).until(4).hashCode(),
-            TimeWindows.of(3).advanceBy(2).grace(2).until(4).hashCode()
-        );
 
         assertNotEquals(
             TimeWindows.of(3).advanceBy(2).grace(1).until(4),
             TimeWindows.of(3).advanceBy(2).grace(2).until(4)
         );
-        assertNotEquals(
-            TimeWindows.of(3).advanceBy(2).grace(1).until(4).hashCode(),
-            TimeWindows.of(3).advanceBy(2).grace(2).until(4).hashCode()
-        );
 
         assertNotEquals(
             TimeWindows.of(3).advanceBy(2).grace(2).until(9),
             TimeWindows.of(3).advanceBy(2).grace(2).until(4)
         );
-        assertNotEquals(
-            TimeWindows.of(3).advanceBy(2).grace(2).until(9).hashCode(),
-            TimeWindows.of(3).advanceBy(2).grace(2).until(4).hashCode()
-        );
     }
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/UnlimitedWindowsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/UnlimitedWindowsTest.java
index 5d139c84541de..a1406547e26b6 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/UnlimitedWindowsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/UnlimitedWindowsTest.java
@@ -21,8 +21,9 @@
 
 import java.util.Map;
 
+import static org.apache.kafka.streams.EqualityCheck.verifyEquality;
+import static org.apache.kafka.streams.EqualityCheck.verifyInEquality;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -51,16 +52,6 @@ public void shouldThrowOnUntil() {
         }
     }
 
-    @Test
-    public void gracePeriodShouldNotBeSettable() {
-        try {
-            UnlimitedWindows.of().grace(0L);
-            fail("should not be able to set grace period");
-        } catch (final IllegalArgumentException e) {
-            // expected
-        }
-    }
-
     @Test
     public void shouldIncludeRecordsThatHappenedOnWindowStart() {
         final UnlimitedWindows w = UnlimitedWindows.of().startOn(anyStartTime);
@@ -88,18 +79,15 @@ public void shouldExcludeRecordsThatHappenedBeforeWindowStart() {
 
     @Test
     public void equalsAndHashcodeShouldBeValidForPositiveCases() {
-        assertEquals(UnlimitedWindows.of(), UnlimitedWindows.of());
-        assertEquals(UnlimitedWindows.of().hashCode(), UnlimitedWindows.of().hashCode());
+        verifyEquality(UnlimitedWindows.of(), UnlimitedWindows.of());
 
-        assertEquals(UnlimitedWindows.of().startOn(1), UnlimitedWindows.of().startOn(1));
-        assertEquals(UnlimitedWindows.of().startOn(1).hashCode(), UnlimitedWindows.of().startOn(1).hashCode());
+        verifyEquality(UnlimitedWindows.of().startOn(1), UnlimitedWindows.of().startOn(1));
 
     }
 
     @Test
     public void equalsAndHashcodeShouldBeValidForNegativeCases() {
-        assertNotEquals(UnlimitedWindows.of().startOn(9), UnlimitedWindows.of().startOn(1));
-        assertNotEquals(UnlimitedWindows.of().startOn(9).hashCode(), UnlimitedWindows.of().startOn(1).hashCode());
+        verifyInEquality(UnlimitedWindows.of().startOn(9), UnlimitedWindows.of().startOn(1));
     }
 
 }
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/WindowsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/WindowsTest.java
index 12ff16612c457..8709031a46831 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/WindowsTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/WindowsTest.java
@@ -21,12 +21,10 @@
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 
 public class WindowsTest {
 
     private class TestWindows extends Windows {
-
         @Override
         public Map windowsFor(final long timestamp) {
             return null;
@@ -36,6 +34,10 @@ public Map windowsFor(final long timestamp) {
         public long size() {
             return 0;
         }
+
+        public long gracePeriodMs() {
+            return 0L;
+        }
     }
 
     @SuppressWarnings("deprecation") // specifically testing deprecated APIs
@@ -58,19 +60,6 @@ public void shouldSetWindowRetentionTime() {
         assertEquals(anyNotNegativeRetentionTime, new TestWindows().until(anyNotNegativeRetentionTime).maintainMs());
     }
 
-
-    @Test
-    public void gracePeriodShouldEnforceBoundaries() {
-        new TestWindows().grace(0L);
-
-        try {
-            new TestWindows().grace(-1L);
-            fail("should not accept negatives");
-        } catch (final IllegalArgumentException e) {
-            //expected
-        }
-    }
-
     @SuppressWarnings("deprecation") // specifically testing deprecated APIs
     @Test(expected = IllegalArgumentException.class)
     public void numberOfSegmentsMustBeAtLeastTwo() {

From 04770916a743ca9526cfa2fdecaef419d429a443 Mon Sep 17 00:00:00 2001
From: Mario Molina 
Date: Fri, 24 Aug 2018 10:32:06 -0500
Subject: [PATCH 0746/1847] KAFKA-5975; No response when deleting topics and
 delete.topic.enable=false (#3960)

This PR implements [KIP-322](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=87295558).

Reviewers: Tom Bentley , Manikumar Reddy O , Jason Gustafson 
---
 .../TopicDeletionDisabledException.java       | 29 +++++++++
 .../apache/kafka/common/protocol/Errors.java  | 41 +++++++------
 .../common/requests/DeleteTopicsRequest.java  |  9 ++-
 .../common/requests/DeleteTopicsResponse.java | 11 +++-
 .../clients/admin/KafkaAdminClientTest.java   | 29 +++++++++
 .../main/scala/kafka/server/KafkaApis.scala   |  6 ++
 ...opicsRequestWithDeletionDisabledTest.scala | 59 +++++++++++++++++++
 7 files changed, 163 insertions(+), 21 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/TopicDeletionDisabledException.java
 create mode 100644 core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala

diff --git a/clients/src/main/java/org/apache/kafka/common/errors/TopicDeletionDisabledException.java b/clients/src/main/java/org/apache/kafka/common/errors/TopicDeletionDisabledException.java
new file mode 100644
index 0000000000000..41577d2a288e7
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/errors/TopicDeletionDisabledException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.common.errors;
+
+public class TopicDeletionDisabledException extends  ApiException {
+    private static final long serialVersionUID = 1L;
+
+    public TopicDeletionDisabledException() {
+    }
+
+    public TopicDeletionDisabledException(String message) {
+        super(message);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index 090dca32651bb..d4610602d1e66 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -80,6 +80,7 @@
 import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.TopicAuthorizationException;
+import org.apache.kafka.common.errors.TopicDeletionDisabledException;
 import org.apache.kafka.common.errors.TopicExistsException;
 import org.apache.kafka.common.errors.TransactionalIdAuthorizationException;
 import org.apache.kafka.common.errors.TransactionCoordinatorFencedException;
@@ -110,7 +111,7 @@
  * Do not add exceptions that occur only on the client or only on the server here.
  */
 public enum Errors {
-    UNKNOWN_SERVER_ERROR(-1, "The server experienced an unexpected error when processing the request",
+    UNKNOWN_SERVER_ERROR(-1, "The server experienced an unexpected error when processing the request,",
             UnknownServerException::new),
     NONE(0, null, message -> null),
     OFFSET_OUT_OF_RANGE(1, "The requested offset is not within the range of offsets maintained by the server.",
@@ -129,7 +130,7 @@ public enum Errors {
             TimeoutException::new),
     BROKER_NOT_AVAILABLE(8, "The broker is not available.",
             BrokerNotAvailableException::new),
-    REPLICA_NOT_AVAILABLE(9, "The replica is not available for the requested topic-partition",
+    REPLICA_NOT_AVAILABLE(9, "The replica is not available for the requested topic-partition.",
             ReplicaNotAvailableException::new),
     MESSAGE_TOO_LARGE(10, "The request included a message larger than the max message size the server will accept.",
             RecordTooLargeException::new),
@@ -161,7 +162,7 @@ public enum Errors {
             "The group member's supported protocols are incompatible with those of existing members" +
                 " or first group member tried to join with empty protocol type or empty protocol list.",
             InconsistentGroupProtocolException::new),
-    INVALID_GROUP_ID(24, "The configured groupId is invalid",
+    INVALID_GROUP_ID(24, "The configured groupId is invalid.",
             InvalidGroupIdException::new),
     UNKNOWN_MEMBER_ID(25, "The coordinator is not aware of this member.",
             UnknownMemberIdException::new),
@@ -171,7 +172,7 @@ public enum Errors {
             InvalidSessionTimeoutException::new),
     REBALANCE_IN_PROGRESS(27, "The group is rebalancing, so a rejoin is needed.",
             RebalanceInProgressException::new),
-    INVALID_COMMIT_OFFSET_SIZE(28, "The committing offset data size is not valid",
+    INVALID_COMMIT_OFFSET_SIZE(28, "The committing offset data size is not valid.",
             InvalidCommitOffsetSizeException::new),
     TOPIC_AUTHORIZATION_FAILED(29, "Topic authorization failed.",
             TopicAuthorizationException::new),
@@ -207,28 +208,28 @@ public enum Errors {
             UnsupportedForMessageFormatException::new),
     POLICY_VIOLATION(44, "Request parameters do not satisfy the configured policy.",
             PolicyViolationException::new),
-    OUT_OF_ORDER_SEQUENCE_NUMBER(45, "The broker received an out of order sequence number",
+    OUT_OF_ORDER_SEQUENCE_NUMBER(45, "The broker received an out of order sequence number.",
             OutOfOrderSequenceException::new),
-    DUPLICATE_SEQUENCE_NUMBER(46, "The broker received a duplicate sequence number",
+    DUPLICATE_SEQUENCE_NUMBER(46, "The broker received a duplicate sequence number.",
             DuplicateSequenceException::new),
     INVALID_PRODUCER_EPOCH(47, "Producer attempted an operation with an old epoch. Either there is a newer producer " +
             "with the same transactionalId, or the producer's transaction has been expired by the broker.",
             ProducerFencedException::new),
-    INVALID_TXN_STATE(48, "The producer attempted a transactional operation in an invalid state",
+    INVALID_TXN_STATE(48, "The producer attempted a transactional operation in an invalid state.",
             InvalidTxnStateException::new),
     INVALID_PRODUCER_ID_MAPPING(49, "The producer attempted to use a producer id which is not currently assigned to " +
-            "its transactional id",
+            "its transactional id.",
             InvalidPidMappingException::new),
     INVALID_TRANSACTION_TIMEOUT(50, "The transaction timeout is larger than the maximum value allowed by " +
                 "the broker (as configured by transaction.max.timeout.ms).",
             InvalidTxnTimeoutException::new),
     CONCURRENT_TRANSACTIONS(51, "The producer attempted to update a transaction " +
-                "while another concurrent operation on the same transaction was ongoing",
+                "while another concurrent operation on the same transaction was ongoing.",
             ConcurrentTransactionsException::new),
     TRANSACTION_COORDINATOR_FENCED(52, "Indicates that the transaction coordinator sending a WriteTxnMarker " +
-            "is no longer the current coordinator for a given producer",
+            "is no longer the current coordinator for a given producer.",
             TransactionCoordinatorFencedException::new),
-    TRANSACTIONAL_ID_AUTHORIZATION_FAILED(53, "Transactional Id authorization failed",
+    TRANSACTIONAL_ID_AUTHORIZATION_FAILED(53, "Transactional Id authorization failed.",
             TransactionalIdAuthorizationException::new),
     SECURITY_DISABLED(54, "Security features are disabled.",
             SecurityDisabledException::new),
@@ -248,7 +249,7 @@ public enum Errors {
             "removed, the producer's metadata is removed from the broker, and future appends by the producer will " +
             "return this exception.",
             UnknownProducerIdException::new),
-    REASSIGNMENT_IN_PROGRESS(60, "A partition reassignment is in progress",
+    REASSIGNMENT_IN_PROGRESS(60, "A partition reassignment is in progress.",
             ReassignmentInProgressException::new),
     DELEGATION_TOKEN_AUTH_DISABLED(61, "Delegation Token feature is not enabled.",
             DelegationTokenDisabledException::new),
@@ -263,19 +264,21 @@ public enum Errors {
             DelegationTokenAuthorizationException::new),
     DELEGATION_TOKEN_EXPIRED(66, "Delegation Token is expired.",
             DelegationTokenExpiredException::new),
-    INVALID_PRINCIPAL_TYPE(67, "Supplied principalType is not supported",
+    INVALID_PRINCIPAL_TYPE(67, "Supplied principalType is not supported.",
             InvalidPrincipalTypeException::new),
-    NON_EMPTY_GROUP(68, "The group is not empty",
+    NON_EMPTY_GROUP(68, "The group is not empty.",
             GroupNotEmptyException::new),
-    GROUP_ID_NOT_FOUND(69, "The group id does not exist",
+    GROUP_ID_NOT_FOUND(69, "The group id does not exist.",
             GroupIdNotFoundException::new),
-    FETCH_SESSION_ID_NOT_FOUND(70, "The fetch session ID was not found",
+    FETCH_SESSION_ID_NOT_FOUND(70, "The fetch session ID was not found.",
             FetchSessionIdNotFoundException::new),
-    INVALID_FETCH_SESSION_EPOCH(71, "The fetch session epoch is invalid",
+    INVALID_FETCH_SESSION_EPOCH(71, "The fetch session epoch is invalid.",
             InvalidFetchSessionEpochException::new),
     LISTENER_NOT_FOUND(72, "There is no listener on the leader broker that matches the listener on which " +
-            "metadata request was processed",
-            ListenerNotFoundException::new),;
+            "metadata request was processed.",
+            ListenerNotFoundException::new),
+    TOPIC_DELETION_DISABLED(73, "Topic deletion is disabled.",
+            TopicDeletionDisabledException::new);
 
     private static final Logger log = LoggerFactory.getLogger(Errors.class);
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
index dbcc25d37c6fa..87f14b4aa6306 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
@@ -51,9 +51,15 @@ public class DeleteTopicsRequest extends AbstractRequest {
      */
     private static final Schema DELETE_TOPICS_REQUEST_V2 = DELETE_TOPICS_REQUEST_V1;
 
+    /**
+     * v3 request is the same that as v2. The response is different based on the request version.
+     * In v3 version a TopicDeletionDisabledException is returned
+     */
+    private static final Schema DELETE_TOPICS_REQUEST_V3 = DELETE_TOPICS_REQUEST_V2;
+
     public static Schema[] schemaVersions() {
         return new Schema[]{DELETE_TOPICS_REQUEST_V0, DELETE_TOPICS_REQUEST_V1,
-            DELETE_TOPICS_REQUEST_V2};
+            DELETE_TOPICS_REQUEST_V2, DELETE_TOPICS_REQUEST_V3};
     }
 
     private final Set topics;
@@ -121,6 +127,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
                 return new DeleteTopicsResponse(topicErrors);
             case 1:
             case 2:
+            case 3:
                 return new DeleteTopicsResponse(throttleTimeMs, topicErrors);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
index db1c4343f9ea0..650caa829ab38 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
@@ -53,8 +53,15 @@ public class DeleteTopicsResponse extends AbstractResponse {
      */
     private static final Schema DELETE_TOPICS_RESPONSE_V2 = DELETE_TOPICS_RESPONSE_V1;
 
+    /**
+     * v3 request is the same that as v2. The response is different based on the request version.
+     * In v3 version a TopicDeletionDisabledException is returned
+     */
+    private static final Schema DELETE_TOPICS_RESPONSE_V3 = DELETE_TOPICS_RESPONSE_V2;
+
     public static Schema[] schemaVersions() {
-        return new Schema[]{DELETE_TOPICS_RESPONSE_V0, DELETE_TOPICS_RESPONSE_V1, DELETE_TOPICS_RESPONSE_V2};
+        return new Schema[]{DELETE_TOPICS_RESPONSE_V0, DELETE_TOPICS_RESPONSE_V1,
+            DELETE_TOPICS_RESPONSE_V2, DELETE_TOPICS_RESPONSE_V3};
     }
 
 
@@ -65,6 +72,8 @@ public static Schema[] schemaVersions() {
      * INVALID_TOPIC_EXCEPTION(17)
      * TOPIC_AUTHORIZATION_FAILED(29)
      * NOT_CONTROLLER(41)
+     * INVALID_REQUEST(42)
+     * TOPIC_DELETION_DISABLED(73)
      */
     private final Map errors;
     private final int throttleTimeMs;
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 7079471402663..0245cbd36950a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -45,6 +45,7 @@
 import org.apache.kafka.common.errors.SaslAuthenticationException;
 import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.TopicDeletionDisabledException;
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.ApiError;
@@ -58,6 +59,8 @@
 import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse;
 import org.apache.kafka.common.requests.DeleteGroupsResponse;
 import org.apache.kafka.common.requests.DeleteRecordsResponse;
+import org.apache.kafka.common.requests.DeleteTopicsRequest;
+import org.apache.kafka.common.requests.DeleteTopicsResponse;
 import org.apache.kafka.common.requests.DescribeAclsResponse;
 import org.apache.kafka.common.requests.DescribeConfigsResponse;
 import org.apache.kafka.common.requests.DescribeGroupsResponse;
@@ -378,6 +381,32 @@ public void testCreateTopicsHandleNotControllerException() throws Exception {
         }
     }
 
+    @Test
+    public void testDeleteTopics() throws Exception {
+        try (AdminClientUnitTestEnv env = mockClientEnv()) {
+            env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+            env.kafkaClient().setNode(env.cluster().controller());
+
+            env.kafkaClient().prepareResponse(body -> body instanceof DeleteTopicsRequest,
+                    new DeleteTopicsResponse(Collections.singletonMap("myTopic", Errors.NONE)));
+            KafkaFuture future = env.adminClient().deleteTopics(Collections.singletonList("myTopic"),
+                    new DeleteTopicsOptions()).all();
+            future.get();
+
+            env.kafkaClient().prepareResponse(body -> body instanceof DeleteTopicsRequest,
+                    new DeleteTopicsResponse(Collections.singletonMap("myTopic", Errors.TOPIC_DELETION_DISABLED)));
+            future = env.adminClient().deleteTopics(Collections.singletonList("myTopic"),
+                    new DeleteTopicsOptions()).all();
+            TestUtils.assertFutureError(future, TopicDeletionDisabledException.class);
+
+            env.kafkaClient().prepareResponse(body -> body instanceof DeleteTopicsRequest,
+                    new DeleteTopicsResponse(Collections.singletonMap("myTopic", Errors.UNKNOWN_TOPIC_OR_PARTITION)));
+            future = env.adminClient().deleteTopics(Collections.singletonList("myTopic"),
+                    new DeleteTopicsOptions()).all();
+            TestUtils.assertFutureError(future, UnknownTopicOrPartitionException.class);
+        }
+    }
+
     @Test
     public void testInvalidTopicNames() throws Exception {
         try (AdminClientUnitTestEnv env = mockClientEnv()) {
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 0273e3da557e3..096ff388bdd46 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1539,6 +1539,12 @@ class KafkaApis(val requestChannel: RequestChannel,
         (topic, Errors.NOT_CONTROLLER)
       }.toMap
       sendResponseCallback(results)
+    } else if (!config.deleteTopicEnable) {
+      val error = if (request.context.apiVersion < 3) Errors.INVALID_REQUEST else Errors.TOPIC_DELETION_DISABLED
+      val results = deleteTopicRequest.topics.asScala.map { topic =>
+        (topic, error)
+      }.toMap
+      sendResponseCallback(results)
     } else {
       // If no authorized topics return immediately
       if (authorizedForDeleteTopics.isEmpty)
diff --git a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala
new file mode 100644
index 0000000000000..20e30c08a9f4e
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestWithDeletionDisabledTest.scala
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.server
+
+import kafka.network.SocketServer
+import kafka.utils._
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.requests.{DeleteTopicsRequest, DeleteTopicsResponse}
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.JavaConverters._
+
+class DeleteTopicsRequestWithDeletionDisabledTest extends BaseRequestTest {
+
+  override def numBrokers: Int = 1
+
+  override def generateConfigs = {
+    val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect,
+      enableControlledShutdown = false, enableDeleteTopic = false,
+      interBrokerSecurityProtocol = Some(securityProtocol),
+      trustStoreFile = trustStoreFile, saslProperties = serverSaslProperties, logDirCount = logDirCount)
+    props.foreach(propertyOverrides)
+    props.map(KafkaConfig.fromProps)
+  }
+
+  @Test
+  def testDeleteRecordsRequest() {
+    val topic = "topic-1"
+    val request = new DeleteTopicsRequest.Builder(Set(topic).asJava, 1000).build()
+    val response = sendDeleteTopicsRequest(request)
+    assertEquals(Errors.TOPIC_DELETION_DISABLED, response.errors.get(topic))
+
+    val v2request = new DeleteTopicsRequest.Builder(Set(topic).asJava, 1000).build(2)
+    val v2response = sendDeleteTopicsRequest(v2request)
+    assertEquals(Errors.INVALID_REQUEST, v2response.errors.get(topic))
+  }
+
+  private def sendDeleteTopicsRequest(request: DeleteTopicsRequest, socketServer: SocketServer = controllerSocketServer): DeleteTopicsResponse = {
+    val response = connectAndSend(request, ApiKeys.DELETE_TOPICS, socketServer)
+    DeleteTopicsResponse.parse(response, request.version)
+  }
+
+}

From 89cf515aecd1ce3a5265a5df94357b3b55694522 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 24 Aug 2018 08:53:54 -0700
Subject: [PATCH 0747/1847] MINOR: replace deprecated remove with delete
 (#5565)

Reviewers: Matthias J. Sax 
---
 .../streams/state/internals/RocksDBSegmentedBytesStore.java   | 2 +-
 .../apache/kafka/streams/state/internals/RocksDBStore.java    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java
index fccb6c1cb592a..fa714651f3469 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java
@@ -237,7 +237,7 @@ Map getWriteBatches(final Collection new WriteBatch());
                     if (record.value == null) {
-                        batch.remove(record.key);
+                        batch.delete(record.key);
                     } else {
                         batch.put(record.key, record.value);
                     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 559f4a527aca9..fbf7df3de8e5c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -283,7 +283,7 @@ void restoreAllInternal(final Collection> records) {
         try (final WriteBatch batch = new WriteBatch()) {
             for (final KeyValue record : records) {
                 if (record.value == null) {
-                    batch.remove(record.key);
+                    batch.delete(record.key);
                 } else {
                     batch.put(record.key, record.value);
                 }
@@ -323,7 +323,7 @@ public void putAll(final List> entries) {
             for (final KeyValue entry : entries) {
                 Objects.requireNonNull(entry.key, "key cannot be null");
                 if (entry.value == null) {
-                    batch.remove(entry.key.get());
+                    batch.delete(entry.key.get());
                 } else {
                     batch.put(entry.key.get(), entry.value);
                 }

From 6f48978b629d7f6eaf3f82783789cf46c3566d9b Mon Sep 17 00:00:00 2001
From: Sam Lendle 
Date: Fri, 24 Aug 2018 11:20:57 -0700
Subject: [PATCH 0748/1847] KAFKA-7240: -total metrics in Streams are incorrect
 (#5467)

Changes:

1. Add org.apache.kafka.streams.processor.internals.metrics.CumulativeCount analogous to Count, but not a SampledStat
2. Use CumulativeCount for -total metrics in streams instead of Count

Testing strategy:

Add a test in StreamsMetricsImplTest which fails on old, incorrect behavior

The contribution is my original work and I license the work to the project under the project's open source license.

Reviewers: Guozhang Wang , John Roesler 
---
 .../processor/internals/StreamTask.java       |  7 +--
 .../processor/internals/StreamThread.java     | 13 +++---
 .../internals/metrics/CumulativeCount.java    | 38 ++++++++++++++++
 .../internals/metrics/StreamsMetricsImpl.java |  2 +-
 .../internals/StreamsMetricsImplTest.java     | 44 +++++++++++++++++++
 5 files changed, 94 insertions(+), 10 deletions(-)
 create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 79df5d158a169..7f3d31fd77607 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -41,6 +41,7 @@
 import org.apache.kafka.streams.processor.Punctuator;
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.TimestampExtractor;
+import org.apache.kafka.streams.processor.internals.metrics.CumulativeCount;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 
@@ -109,7 +110,7 @@ protected static final class TaskMetrics {
             );
             parent.add(
                 new MetricName("commit-total", group, "The total number of occurrence of commit operations.", allTagMap),
-                new Count()
+                new CumulativeCount()
             );
 
             // add the operation metrics with additional tags
@@ -129,7 +130,7 @@ protected static final class TaskMetrics {
             );
             taskCommitTimeSensor.add(
                 new MetricName("commit-total", group, "The total number of occurrence of commit operations.", tagMap),
-                new Count()
+                new CumulativeCount()
             );
 
             // add the metrics for enforced processing
@@ -140,7 +141,7 @@ protected static final class TaskMetrics {
             );
             taskEnforcedProcessSensor.add(
                     new MetricName("enforced-process-total", group, "The total number of occurrence of enforced-process operations.", tagMap),
-                    new Count()
+                    new CumulativeCount()
             );
 
         }
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index efd94eaf6373f..28cedbe41976b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -44,6 +44,7 @@
 import org.apache.kafka.streams.processor.TaskId;
 import org.apache.kafka.streams.processor.TaskMetadata;
 import org.apache.kafka.streams.processor.ThreadMetadata;
+import org.apache.kafka.streams.processor.internals.metrics.CumulativeCount;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.apache.kafka.streams.state.internals.ThreadCache;
 import org.slf4j.Logger;
@@ -437,7 +438,7 @@ StreamTask createTask(final Consumer consumer,
                 cache,
                 time,
                 () -> createProducer(taskId),
-                streamsMetrics.tasksClosedSensor);
+                streamsMetrics.taskClosedSensor);
         }
 
         private Producer createProducer(final TaskId id) {
@@ -518,7 +519,7 @@ static class StreamsMetricsThreadImpl extends StreamsMetricsImpl {
         private final Sensor processTimeSensor;
         private final Sensor punctuateTimeSensor;
         private final Sensor taskCreatedSensor;
-        private final Sensor tasksClosedSensor;
+        private final Sensor taskClosedSensor;
 
         StreamsMetricsThreadImpl(final Metrics metrics, final String threadName) {
             super(metrics, threadName);
@@ -532,7 +533,7 @@ static class StreamsMetricsThreadImpl extends StreamsMetricsImpl {
             addAvgMaxLatency(pollTimeSensor, group, tagMap(), "poll");
             // can't use addInvocationRateAndCount due to non-standard description string
             pollTimeSensor.add(metrics.metricName("poll-rate", group, "The average per-second number of record-poll calls", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
-            pollTimeSensor.add(metrics.metricName("poll-total", group, "The total number of record-poll calls", tagMap()), new Count());
+            pollTimeSensor.add(metrics.metricName("poll-total", group, "The total number of record-poll calls", tagMap()), new CumulativeCount());
 
             processTimeSensor = threadLevelSensor("process-latency", Sensor.RecordingLevel.INFO);
             addAvgMaxLatency(processTimeSensor, group, tagMap(), "process");
@@ -546,9 +547,9 @@ static class StreamsMetricsThreadImpl extends StreamsMetricsImpl {
             taskCreatedSensor.add(metrics.metricName("task-created-rate", "stream-metrics", "The average per-second number of newly created tasks", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
             taskCreatedSensor.add(metrics.metricName("task-created-total", "stream-metrics", "The total number of newly created tasks", tagMap()), new Total());
 
-            tasksClosedSensor = threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
-            tasksClosedSensor.add(metrics.metricName("task-closed-rate", group, "The average per-second number of closed tasks", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
-            tasksClosedSensor.add(metrics.metricName("task-closed-total", group, "The total number of closed tasks", tagMap()), new Total());
+            taskClosedSensor = threadLevelSensor("task-closed", Sensor.RecordingLevel.INFO);
+            taskClosedSensor.add(metrics.metricName("task-closed-rate", group, "The average per-second number of closed tasks", tagMap()), new Rate(TimeUnit.SECONDS, new Count()));
+            taskClosedSensor.add(metrics.metricName("task-closed-total", group, "The total number of closed tasks", tagMap()), new Total());
         }
     }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java
new file mode 100644
index 0000000000000..2c12c2b6e9d41
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/CumulativeCount.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.metrics;
+
+import org.apache.kafka.common.metrics.MeasurableStat;
+import org.apache.kafka.common.metrics.MetricConfig;
+
+/**
+ * A non-SampledStat version of Count for measuring -total metrics in streams
+ */
+public class CumulativeCount implements MeasurableStat {
+
+    private double count = 0.0;
+
+    @Override
+    public void record(final MetricConfig config, final double value, final long timeMs) {
+        count += 1;
+    }
+
+    @Override
+    public double measure(final MetricConfig config, final long now) {
+        return count;
+    }
+}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
index 56166a4d372a4..170311238ecf3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/metrics/StreamsMetricsImpl.java
@@ -398,7 +398,7 @@ public static void addInvocationRateAndCount(final Sensor sensor,
                 "The total number of occurrence of " + operation + " operations.",
                 tags
             ),
-            new Count()
+            new CumulativeCount()
         );
     }
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImplTest.java
index b065e2ca7b51c..7ce27b4b6d668 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetricsImplTest.java
@@ -17,11 +17,17 @@
 package org.apache.kafka.streams.processor.internals;
 
 
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.metrics.KafkaMetric;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
 import org.junit.Test;
 
+import java.util.concurrent.TimeUnit;
+
 import static org.junit.Assert.assertEquals;
 
 public class StreamsMetricsImplTest {
@@ -96,4 +102,42 @@ public void testThroughputMetrics() {
         streamsMetrics.removeSensor(sensor1);
         assertEquals(defaultMetrics, streamsMetrics.metrics().size());
     }
+
+    @Test
+    public void testTotalMetricDoesntDecrease() {
+        final MockTime time = new MockTime(1);
+        final MetricConfig config = new MetricConfig().timeWindow(1, TimeUnit.MILLISECONDS);
+        final Metrics metrics = new Metrics(config, time);
+        final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "");
+
+        final String scope = "scope";
+        final String entity = "entity";
+        final String operation = "op";
+
+        final Sensor sensor = streamsMetrics.addLatencyAndThroughputSensor(
+                scope,
+                entity,
+                operation,
+                Sensor.RecordingLevel.INFO
+        );
+
+        final double latency = 100.0;
+        final MetricName totalMetricName = metrics.metricName(
+                "op-total",
+                "stream-scope-metrics",
+                "",
+                "client-id",
+                "",
+                "scope-id",
+                "entity"
+        );
+
+        final KafkaMetric totalMetric = metrics.metric(totalMetricName);
+
+        for (int i = 0; i < 10; i++) {
+            assertEquals(i, Math.round(totalMetric.measurable().measure(config, time.milliseconds())));
+            sensor.record(latency, time.milliseconds());
+        }
+
+    }
 }

From 42610e5aa105339b4f691ca97e300202b7d4812b Mon Sep 17 00:00:00 2001
From: Gantigmaa Selenge <39860586+tinaselenge@users.noreply.github.com>
Date: Fri, 24 Aug 2018 22:03:58 +0100
Subject: [PATCH 0749/1847] MINOR: Fix broken link to record format in protocol
 docs (#5526)

Co-authored-by: Mickael Maison 
Co-authored-by: Gantigmaa Selenge  

Reviewers: Sriharsha Chintalapani , Jason Gustafson 
---
 docs/protocol.html | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/protocol.html b/docs/protocol.html
index 416823aa588e8..b884992a7bcdd 100644
--- a/docs/protocol.html
+++ b/docs/protocol.html
@@ -39,7 +39,7 @@ 
      Kafka protocol guide
      
             
    • Protocol Primitive Types
             
    • Notes on reading the request format grammars
             
    • Common Request and Response Structure
-            
    • Message Sets
+            
    • Record Batch
         
    
     
     
  • Constants
@@ -56,7 +56,7 @@ 
    Preliminaries<
 
 
    Network
    
 
-
    Kafka uses a binary protocol over TCP. The protocol defines all apis as request response message pairs. All messages are size delimited and are made up of the following primitive types.
    
+
    Kafka uses a binary protocol over TCP. The protocol defines all APIs as request response message pairs. All messages are size delimited and are made up of the following primitive types.
    
 
 
    The client initiates a socket connection and then writes a sequence of request messages and reads back the corresponding response message. No handshake is required on connection or disconnection. TCP is happier if you maintain persistent connections used for many requests to amortize the cost of the TCP handshake, but beyond this penalty connecting is pretty cheap.
    
 
@@ -76,11 +76,11 @@ 
    Partitioning and
 
 
    How can the client find out which topics exist, what partitions they have, and which brokers currently host those partitions so that it can direct its requests to the right hosts? This information is dynamic, so you can't just configure each client with some static mapping file. Instead all Kafka brokers can answer a metadata request that describes the current state of the cluster: what topics there are, which partitions those topics have, which broker is the leader for those partitions, and the host and port information for these brokers.
    
 
-
    In other words, the client needs to somehow find one broker and that broker will tell the client about all the other brokers that exist and what partitions they host. This first broker may itself go down so the best practice for a client implementation is to take a list of two or three urls to bootstrap from. The user can then choose to use a load balancer or just statically configure two or three of their kafka hosts in the clients.
    
+
    In other words, the client needs to somehow find one broker and that broker will tell the client about all the other brokers that exist and what partitions they host. This first broker may itself go down so the best practice for a client implementation is to take a list of two or three URLs to bootstrap from. The user can then choose to use a load balancer or just statically configure two or three of their Kafka hosts in the clients.
    
 
 
    The client does not need to keep polling to see if the cluster has changed; it can fetch metadata once when it is instantiated cache that metadata until it receives an error indicating that the metadata is out of date. This error can come in two forms: (1) a socket error indicating the client cannot communicate with a particular broker, (2) an error code in the response to a request indicating that this broker no longer hosts the partition for which data was requested.
    
 
      
-    
    1. Cycle through a list of "bootstrap" kafka urls until we find one we can connect to. Fetch cluster metadata.
      2. 
+    
    2. Cycle through a list of "bootstrap" Kafka URLs until we find one we can connect to. Fetch cluster metadata.
      3. 
     
    3. Process fetch or produce requests, directing them to the appropriate broker based on the topic/partitions they send to or fetch from.
      4. 
     
    4. If we get an appropriate error, refresh the metadata and try again.
      5. 
 
    
@@ -103,7 +103,7 @@ 
    Batching
    
 
-
    Our apis encourage batching small things together for efficiency. We have found this is a very significant performance win. Both our API to send messages and our API to fetch messages always work with a sequence of messages not a single message to encourage this. A clever client can make use of this and support an "asynchronous" mode in which it batches together messages sent individually and sends them in larger clumps. We go even further with this and allow the batching across multiple topics and partitions, so a produce request may contain data to append to many partitions and a fetch request may pull data from many partitions all at once.
    
+
    Our APIs encourage batching small things together for efficiency. We have found this is a very significant performance win. Both our API to send messages and our API to fetch messages always work with a sequence of messages not a single message to encourage this. A clever client can make use of this and support an "asynchronous" mode in which it batches together messages sent individually and sends them in larger clumps. We go even further with this and allow the batching across multiple topics and partitions, so a produce request may contain data to append to many partitions and a fetch request may pull data from many partitions all at once.
    
 
 
    The client implementer can choose to ignore this and send everything one at a time if they like.
    
 

From 50a145d090ae6a4400000d844bf708e6e68a5dce Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Sat, 25 Aug 2018 21:19:41 +0530
Subject: [PATCH 0750/1847] MINOR: Remove unused compressionType parameter from
 TestUtils.produceMessages (#5569)

Reviewers: Ismael Juma 
---
 core/src/test/scala/unit/kafka/utils/TestUtils.scala | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 2ca3a6c986dc8..48d7d3fd199ff 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -916,10 +916,7 @@ object TestUtils extends Logging {
 
   def produceMessages(servers: Seq[KafkaServer],
                       records: Seq[ProducerRecord[Array[Byte], Array[Byte]]],
-                      acks: Int = -1,
-                      compressionType: CompressionType = CompressionType.NONE): Unit = {
-    val props = new Properties()
-    props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType.name)
+                      acks: Int = -1): Unit = {
     val producer = createProducer(TestUtils.getBrokerListStrFromServers(servers), acks = acks)
     try {
       val futures = records.map(producer.send)
@@ -935,11 +932,10 @@ object TestUtils extends Logging {
   def generateAndProduceMessages(servers: Seq[KafkaServer],
                                  topic: String,
                                  numMessages: Int,
-                                 acks: Int = -1,
-                                 compressionType: CompressionType = CompressionType.NONE): Seq[String] = {
+                                 acks: Int = -1): Seq[String] = {
     val values = (0 until numMessages).map(x =>  s"test-$x")
     val records = values.map(v => new ProducerRecord[Array[Byte], Array[Byte]](topic, v.getBytes))
-    produceMessages(servers, records, acks, compressionType)
+    produceMessages(servers, records, acks)
     values
   }
 

From e8e3e34c67c45ae76a51758c600ddb4f7e707717 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sat, 25 Aug 2018 08:55:15 -0700
Subject: [PATCH 0751/1847] KAFKA-7309; Upgrade Jacoco for Java 11 support

Jacoco 0.8.2 adds Java 11 support:

https://github.com/jacoco/jacoco/releases/tag/v0.8.2

Java 11 RC1 is out so it would be good for us to
get a working CI build.

Author: Ismael Juma 

Reviewers: Dong Lin 

Closes #5568 from ijuma/jacoco-0.8.2
---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index c963253a8bca9..9b4610d0b2274 100644
--- a/build.gradle
+++ b/build.gradle
@@ -385,7 +385,7 @@ subprojects {
     apply plugin: "jacoco"
 
     jacoco {
-      toolVersion = "0.8.1"
+      toolVersion = "0.8.2"
     }
 
     // NOTE: Jacoco Gradle plugin does not support "offline instrumentation" this means that classes mocked by PowerMock

From 93a60ee266a202c0b1e94010ce72ba7eb8d068a5 Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Sun, 26 Aug 2018 03:08:25 +0800
Subject: [PATCH 0752/1847] KAFKA-7139; Support option to exclude the internal
 topics in kafka-topics.sh (#5349)

This patch implements KIP-338: https://cwiki.apache.org/confluence/display/KAFKA/KIP-338+Support+to+exclude+the+internal+topics+in+kafka-topics.sh+command.

Reviewers: Andras Beni , Jason Gustafson 
---
 .../main/scala/kafka/admin/TopicCommand.scala |  8 +++++--
 .../unit/kafka/admin/TopicCommandTest.scala   | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/TopicCommand.scala b/core/src/main/scala/kafka/admin/TopicCommand.scala
index 3914b564eda3f..8d89f1050f4f5 100755
--- a/core/src/main/scala/kafka/admin/TopicCommand.scala
+++ b/core/src/main/scala/kafka/admin/TopicCommand.scala
@@ -83,12 +83,13 @@ object TopicCommand extends Logging {
 
   private def getTopics(zkClient: KafkaZkClient, opts: TopicCommandOptions): Seq[String] = {
     val allTopics = zkClient.getAllTopicsInCluster.sorted
+    val excludeInternalTopics = opts.options.has(opts.excludeInternalTopicOpt)
     if (opts.options.has(opts.topicOpt)) {
       val topicsSpec = opts.options.valueOf(opts.topicOpt)
       val topicsFilter = new Whitelist(topicsSpec)
-      allTopics.filter(topicsFilter.isTopicAllowed(_, excludeInternalTopics = false))
+      allTopics.filter(topicsFilter.isTopicAllowed(_, excludeInternalTopics))
     } else
-      allTopics
+      allTopics.filterNot(Topic.isInternal(_) && excludeInternalTopics)
   }
 
   def createTopic(zkClient: KafkaZkClient, opts: TopicCommandOptions) {
@@ -355,6 +356,8 @@ object TopicCommand extends Logging {
 
     val forceOpt = parser.accepts("force", "Suppress console prompts")
 
+    val excludeInternalTopicOpt = parser.accepts("exclude-internal", "exclude internal topics when running list or describe command. The internal topics will be listed by default")
+
     val options = parser.parse(args : _*)
 
     val allTopicLevelOpts: Set[OptionSpec[_]] = Set(alterOpt, createOpt, describeOpt, listOpt, deleteOpt)
@@ -381,6 +384,7 @@ object TopicCommand extends Logging {
         allTopicLevelOpts -- Set(describeOpt) + reportUnderReplicatedPartitionsOpt + reportUnavailablePartitionsOpt)
       CommandLineUtils.checkInvalidArgs(parser, options, ifExistsOpt, allTopicLevelOpts -- Set(alterOpt, deleteOpt))
       CommandLineUtils.checkInvalidArgs(parser, options, ifNotExistsOpt, allTopicLevelOpts -- Set(createOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, excludeInternalTopicOpt, allTopicLevelOpts -- Set(listOpt, describeOpt))
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
index 782fcf539f3f1..13f23e69b5302 100644
--- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala
@@ -242,4 +242,28 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging with RackAwareT
     }
   }
 
+  @Test
+  def testDescribeAndListTopicsWithoutInternalTopics() {
+    val brokers = List(0)
+    val topic = "testDescribeAndListTopicsWithoutInternalTopics"
+    TestUtils.createBrokersInZk(zkClient, brokers)
+
+    TopicCommand.createTopic(zkClient,
+      new TopicCommandOptions(Array("--partitions", "1", "--replication-factor", "1", "--topic", topic)))
+    // create a internal topic
+    TopicCommand.createTopic(zkClient,
+      new TopicCommandOptions(Array("--partitions", "1", "--replication-factor", "1", "--topic", Topic.GROUP_METADATA_TOPIC_NAME)))
+
+    // test describe
+    var output = TestUtils.grabConsoleOutput(TopicCommand.describeTopic(zkClient,
+      new TopicCommandOptions(Array("--describe", "--exclude-internal"))))
+    assertTrue(output.contains(topic))
+    assertFalse(output.contains(Topic.GROUP_METADATA_TOPIC_NAME))
+
+    // test list
+    output = TestUtils.grabConsoleOutput(TopicCommand.listTopics(zkClient,
+      new TopicCommandOptions(Array("--list", "--exclude-internal"))))
+    assertTrue(output.contains(topic))
+    assertFalse(output.contains(Topic.GROUP_METADATA_TOPIC_NAME))
+  }
 }

From ebd1354fd7578deb1cfb15846d7dbf8c9cf26074 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Mon, 27 Aug 2018 17:39:44 -0700
Subject: [PATCH 0753/1847] KAFKA-7347; Return not leader error for
 OffsetsForLeaderEpoch requests to non-replicas (#5576)

The broker should return NOT_LEADER_FOR_PARTITION for OffsetsForLeaderEpoch requests against non-replicas instead of UNKNOWN_TOPIC_OR_PARTITION. This patch also fixes a minor bug in the handling of ListOffsets request using the DEBUG replica id. We should return UNKNOWN_TOPIC_OR_PARTITION if the topic doesn't exist.

Reviewers: Jun Rao 
---
 .../scala/kafka/server/ReplicaManager.scala   | 19 ++--
 .../api/AdminClientIntegrationTest.scala      |  2 +-
 .../AlterReplicaLogDirsRequestTest.scala      |  4 +-
 .../kafka/server/ListOffsetsRequestTest.scala | 88 +++++++++++++++++++
 .../OffsetsForLeaderEpochRequestTest.scala    | 65 ++++++++++++++
 .../epoch/LeaderEpochIntegrationTest.scala    |  2 +-
 6 files changed, 171 insertions(+), 9 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala
 create mode 100644 core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala

diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 14e537e63db2d..59581e738cf1f 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -35,7 +35,6 @@ import org.apache.kafka.common.errors._
 import org.apache.kafka.common.internals.Topic
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
-import org.apache.kafka.common.protocol.Errors.{KAFKA_STORAGE_ERROR, UNKNOWN_TOPIC_OR_PARTITION}
 import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction
 import org.apache.kafka.common.requests.DescribeLogDirsResponse.{LogDirInfo, ReplicaInfo}
@@ -405,12 +404,18 @@ class ReplicaManager(val config: KafkaConfig,
         else
           partition.getReplica(brokerId).getOrElse(
             throw new ReplicaNotAvailableException(s"Replica $brokerId is not available for partition $topicPartition"))
-      case None =>
+
+      case None if metadataCache.contains(topicPartition) =>
         throw new ReplicaNotAvailableException(s"Replica $brokerId is not available for partition $topicPartition")
+
+      case None =>
+        throw new UnknownTopicOrPartitionException(s"Partition $topicPartition doesn't exist")
     }
   }
 
-  def getReplicaOrException(topicPartition: TopicPartition): Replica = getReplicaOrException(topicPartition, localBrokerId)
+  def getReplicaOrException(topicPartition: TopicPartition): Replica = {
+    getReplicaOrException(topicPartition, localBrokerId)
+  }
 
   def getLeaderReplicaIfLocal(topicPartition: TopicPartition): Replica =  {
     val (_, replica) = getPartitionAndLeaderReplicaIfLocal(topicPartition)
@@ -1475,11 +1480,15 @@ class ReplicaManager(val config: KafkaConfig,
       val epochEndOffset = getPartition(tp) match {
         case Some(partition) =>
           if (partition eq ReplicaManager.OfflinePartition)
-            new EpochEndOffset(KAFKA_STORAGE_ERROR, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
+            new EpochEndOffset(Errors.KAFKA_STORAGE_ERROR, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
           else
             partition.lastOffsetForLeaderEpoch(leaderEpoch)
+
+        case None if metadataCache.contains(tp) =>
+          new EpochEndOffset(Errors.NOT_LEADER_FOR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
+
         case None =>
-          new EpochEndOffset(UNKNOWN_TOPIC_OR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
+          new EpochEndOffset(Errors.UNKNOWN_TOPIC_OR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
       }
       tp -> epochEndOffset
     }
diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
index 20cf03829cbcf..672a6698c3fa1 100644
--- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala
@@ -319,7 +319,7 @@ class AdminClientIntegrationTest extends IntegrationTestHarness with Logging {
       new AlterReplicaLogDirsOptions).values.asScala.values
     futures.foreach { future =>
       val exception = intercept[ExecutionException](future.get)
-      assertTrue(exception.getCause.isInstanceOf[ReplicaNotAvailableException])
+      assertTrue(exception.getCause.isInstanceOf[UnknownTopicOrPartitionException])
     }
 
     createTopic(topic, numPartitions = 1, replicationFactor = serverCount)
diff --git a/core/src/test/scala/unit/kafka/server/AlterReplicaLogDirsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AlterReplicaLogDirsRequestTest.scala
index e6fa1cb7d6e9d..e986805e2fd96 100644
--- a/core/src/test/scala/unit/kafka/server/AlterReplicaLogDirsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AlterReplicaLogDirsRequestTest.scala
@@ -49,7 +49,7 @@ class AlterReplicaLogDirsRequestTest extends BaseRequestTest {
     // The response should show error REPLICA_NOT_AVAILABLE for all partitions
     (0 until partitionNum).foreach { partition =>
       val tp = new TopicPartition(topic, partition)
-      assertEquals(Errors.REPLICA_NOT_AVAILABLE, alterReplicaLogDirsResponse1.responses().get(tp))
+      assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, alterReplicaLogDirsResponse1.responses().get(tp))
       assertTrue(servers.head.logManager.getLog(tp).isEmpty)
     }
 
@@ -85,7 +85,7 @@ class AlterReplicaLogDirsRequestTest extends BaseRequestTest {
     partitionDirs1.put(new TopicPartition(topic, 1), validDir1)
     val alterReplicaDirResponse1 = sendAlterReplicaLogDirsRequest(partitionDirs1.toMap)
     assertEquals(Errors.LOG_DIR_NOT_FOUND, alterReplicaDirResponse1.responses().get(new TopicPartition(topic, 0)))
-    assertEquals(Errors.REPLICA_NOT_AVAILABLE, alterReplicaDirResponse1.responses().get(new TopicPartition(topic, 1)))
+    assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, alterReplicaDirResponse1.responses().get(new TopicPartition(topic, 1)))
 
     createTopic(topic, 3, 1)
 
diff --git a/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala
new file mode 100644
index 0000000000000..6ee47eecda2ff
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.server
+
+import java.lang.{Long => JLong}
+
+import kafka.utils.TestUtils
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.requests.{IsolationLevel, ListOffsetRequest, ListOffsetResponse}
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.JavaConverters._
+
+class ListOffsetsRequestTest extends BaseRequestTest {
+
+  @Test
+  def testListOffsetsErrorCodes(): Unit = {
+    val topic = "topic"
+    val partition = new TopicPartition(topic, 0)
+
+    val consumerRequest = ListOffsetRequest.Builder
+      .forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
+      .setTargetTimes(Map(partition -> ListOffsetRequest.EARLIEST_TIMESTAMP.asInstanceOf[JLong]).asJava)
+      .build()
+
+    val replicaRequest = ListOffsetRequest.Builder
+      .forReplica(ApiKeys.LIST_OFFSETS.latestVersion, servers.head.config.brokerId)
+      .setTargetTimes(Map(partition -> ListOffsetRequest.EARLIEST_TIMESTAMP.asInstanceOf[JLong]).asJava)
+      .build()
+
+    val debugReplicaRequest = ListOffsetRequest.Builder
+      .forReplica(ApiKeys.LIST_OFFSETS.latestVersion, ListOffsetRequest.DEBUGGING_REPLICA_ID)
+      .setTargetTimes(Map(partition -> ListOffsetRequest.EARLIEST_TIMESTAMP.asInstanceOf[JLong]).asJava)
+      .build()
+
+    // Unknown topic
+    val randomBrokerId = servers.head.config.brokerId
+    assertResponseError(Errors.UNKNOWN_TOPIC_OR_PARTITION, randomBrokerId, consumerRequest)
+    assertResponseError(Errors.UNKNOWN_TOPIC_OR_PARTITION, randomBrokerId, replicaRequest)
+    assertResponseError(Errors.UNKNOWN_TOPIC_OR_PARTITION, randomBrokerId, debugReplicaRequest)
+
+    val partitionToLeader = TestUtils.createTopic(zkClient, topic, numPartitions = 1, replicationFactor = 2, servers)
+    val replicas = zkClient.getReplicasForPartition(partition).toSet
+    val leader = partitionToLeader(partition.partition)
+    val follower = replicas.find(_ != leader).get
+    val nonReplica = servers.map(_.config.brokerId).find(!replicas.contains(_)).get
+
+    // Follower
+    assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, follower, consumerRequest)
+    assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, follower, replicaRequest)
+    assertResponseError(Errors.NONE, follower, debugReplicaRequest)
+
+    // Non-replica
+    assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, nonReplica, consumerRequest)
+    assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, nonReplica, replicaRequest)
+    assertResponseError(Errors.REPLICA_NOT_AVAILABLE, nonReplica, debugReplicaRequest)
+  }
+
+  private def assertResponseError(error: Errors, brokerId: Int, request: ListOffsetRequest): Unit = {
+    val response = sendRequest(brokerId, request)
+    assertEquals(request.partitionTimestamps.size, response.responseData.size)
+    response.responseData.asScala.values.foreach { partitionData =>
+      assertEquals(error, partitionData.error)
+    }
+  }
+
+  private def sendRequest(leaderId: Int, request: ListOffsetRequest): ListOffsetResponse = {
+    val response = connectAndSend(request, ApiKeys.LIST_OFFSETS, destination = brokerSocketServer(leaderId))
+    ListOffsetResponse.parse(response, request.version)
+  }
+
+}
diff --git a/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala b/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala
new file mode 100644
index 0000000000000..c6385f325ece4
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.server
+
+import kafka.utils.TestUtils
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.requests.{OffsetsForLeaderEpochRequest, OffsetsForLeaderEpochResponse}
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.JavaConverters._
+
+class OffsetsForLeaderEpochRequestTest extends BaseRequestTest {
+
+  @Test
+  def testOffsetsForLeaderEpochErrorCodes(): Unit = {
+    val topic = "topic"
+    val partition = new TopicPartition(topic, 0)
+
+    val request = new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion)
+      .add(partition, 0)
+      .build()
+
+    // Unknown topic
+    val randomBrokerId = servers.head.config.brokerId
+    assertResponseError(Errors.UNKNOWN_TOPIC_OR_PARTITION, randomBrokerId, request)
+
+    val partitionToLeader = TestUtils.createTopic(zkClient, topic, numPartitions = 1, replicationFactor = 2, servers)
+    val replicas = zkClient.getReplicasForPartition(partition).toSet
+    val leader = partitionToLeader(partition.partition)
+    val follower = replicas.find(_ != leader).get
+    val nonReplica = servers.map(_.config.brokerId).find(!replicas.contains(_)).get
+
+    assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, follower, request)
+    assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, nonReplica, request)
+  }
+
+  private def assertResponseError(error: Errors, brokerId: Int, request: OffsetsForLeaderEpochRequest): Unit = {
+    val response = sendRequest(brokerId, request)
+    assertEquals(request.epochsByTopicPartition.size, response.responses.size)
+    response.responses.asScala.values.foreach { partitionData =>
+      assertEquals(error, partitionData.error)
+    }
+  }
+
+  private def sendRequest(leaderId: Int, request: OffsetsForLeaderEpochRequest): OffsetsForLeaderEpochResponse = {
+    val response = connectAndSend(request, ApiKeys.OFFSET_FOR_LEADER_EPOCH, destination = brokerSocketServer(leaderId))
+    OffsetsForLeaderEpochResponse.parse(response, request.version)
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index a6b7732581114..17683f4f3fd53 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -128,7 +128,7 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
 
     //And should get no leader for partition error from t1p1 (as it's not on broker 0)
     assertTrue(offsetsForEpochs(t1p1).hasError)
-    assertEquals(UNKNOWN_TOPIC_OR_PARTITION, offsetsForEpochs(t1p1).error)
+    assertEquals(NOT_LEADER_FOR_PARTITION, offsetsForEpochs(t1p1).error)
     assertEquals(UNDEFINED_EPOCH_OFFSET, offsetsForEpochs(t1p1).endOffset)
 
     //Repointing to broker 1 we should get the correct offset for t1p1

From 7d2b95895ae53060f5743a5e4b8f06ddc039bf66 Mon Sep 17 00:00:00 2001
From: Joan Goyeau 
Date: Tue, 28 Aug 2018 02:44:12 +0100
Subject: [PATCH 0754/1847] MINOR: Correct folder for package object scala
 (#5573)

Reviewers: Guozhang Wang , John Roesler 
---
 .../main/scala/org/apache/kafka/streams/{ => scala}/package.scala | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename streams/streams-scala/src/main/scala/org/apache/kafka/streams/{ => scala}/package.scala (100%)

diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/package.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/package.scala
similarity index 100%
rename from streams/streams-scala/src/main/scala/org/apache/kafka/streams/package.scala
rename to streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/package.scala

From 70c51633f2f4bdcdc045697c17cbb47c78ed14ad Mon Sep 17 00:00:00 2001
From: Anna Povzner 
Date: Tue, 28 Aug 2018 10:45:00 -0700
Subject: [PATCH 0755/1847] KAFKA-7128; Follower has to catch up to offset
 within current leader epoch to join ISR (#5557)

If follower is not in ISR, it has to fetch up to start offset of the current leader epoch. Otherwise we risk losing committed data. Added unit test to verify this behavior.

Reviewers: Jason Gustafson 
---
 .../main/scala/kafka/cluster/Partition.scala  | 24 ++++--
 .../unit/kafka/cluster/PartitionTest.scala    | 86 ++++++++++++++++++-
 2 files changed, 103 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index a92340f2a4bb6..22c1508bb8ca0 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -62,6 +62,9 @@ class Partition(val topic: String,
   private val leaderIsrUpdateLock = new ReentrantReadWriteLock
   private var zkVersion: Int = LeaderAndIsr.initialZKVersion
   @volatile private var leaderEpoch: Int = LeaderAndIsr.initialLeaderEpoch - 1
+  // start offset for 'leaderEpoch' above (leader epoch of the current leader for this partition),
+  // defined when this broker is leader for partition
+  @volatile private var leaderEpochStartOffsetOpt: Option[Long] = None
   @volatile var leaderReplicaIdOpt: Option[Int] = None
   @volatile var inSyncReplicas: Set[Replica] = Set.empty[Replica]
 
@@ -263,6 +266,7 @@ class Partition(val topic: String,
       allReplicasMap.clear()
       inSyncReplicas = Set.empty[Replica]
       leaderReplicaIdOpt = None
+      leaderEpochStartOffsetOpt = None
       removePartitionMetrics()
       logManager.asyncDelete(topicPartition)
       logManager.asyncDelete(topicPartition, isFuture = true)
@@ -287,18 +291,19 @@ class Partition(val topic: String,
       // remove assigned replicas that have been removed by the controller
       (assignedReplicas.map(_.brokerId) -- newAssignedReplicas).foreach(removeReplica)
       inSyncReplicas = newInSyncReplicas
+      newAssignedReplicas.foreach(id => getOrCreateReplica(id, partitionStateInfo.isNew))
 
+      val leaderReplica = getReplica().get
+      val leaderEpochStartOffset = leaderReplica.logEndOffset.messageOffset
       info(s"$topicPartition starts at Leader Epoch ${partitionStateInfo.basePartitionState.leaderEpoch} from " +
-        s"offset ${getReplica().get.logEndOffset.messageOffset}. Previous Leader Epoch was: $leaderEpoch")
+        s"offset $leaderEpochStartOffset. Previous Leader Epoch was: $leaderEpoch")
 
       //We cache the leader epoch here, persisting it only if it's local (hence having a log dir)
       leaderEpoch = partitionStateInfo.basePartitionState.leaderEpoch
-      newAssignedReplicas.foreach(id => getOrCreateReplica(id, partitionStateInfo.isNew))
-
+      leaderEpochStartOffsetOpt = Some(leaderEpochStartOffset)
       zkVersion = partitionStateInfo.basePartitionState.zkVersion
       val isNewLeader = leaderReplicaIdOpt.map(_ != localBrokerId).getOrElse(true)
 
-      val leaderReplica = getReplica().get
       val curLeaderLogEndOffset = leaderReplica.logEndOffset.messageOffset
       val curTimeMs = time.milliseconds
       // initialize lastCaughtUpTime of replicas as well as their lastFetchTimeMs and lastFetchLeaderLogEndOffset.
@@ -344,6 +349,7 @@ class Partition(val topic: String,
       (assignedReplicas.map(_.brokerId) -- newAssignedReplicas).foreach(removeReplica)
       inSyncReplicas = Set.empty[Replica]
       leaderEpoch = partitionStateInfo.basePartitionState.leaderEpoch
+      leaderEpochStartOffsetOpt = None
       zkVersion = partitionStateInfo.basePartitionState.zkVersion
 
       // If the leader is unchanged and the epochs are no more than one change apart, indicate that no follower changes are required
@@ -388,7 +394,11 @@ class Partition(val topic: String,
 
   /**
    * Check and maybe expand the ISR of the partition.
-   * A replica will be added to ISR if its LEO >= current hw of the partition.
+   * A replica will be added to ISR if its LEO >= current hw of the partition and it is caught up to
+   * an offset within the current leader epoch. A replica must be caught up to the current leader
+   * epoch before it can join ISR, because otherwise, if there is committed data between current
+   * leader's HW and LEO, the replica may become the leader before it fetches the committed data
+   * and the data will be lost.
    *
    * Technically, a replica shouldn't be in ISR if it hasn't caught up for longer than replicaLagTimeMaxMs,
    * even if its log end offset is >= HW. However, to be consistent with how the follower determines
@@ -405,9 +415,11 @@ class Partition(val topic: String,
         case Some(leaderReplica) =>
           val replica = getReplica(replicaId).get
           val leaderHW = leaderReplica.highWatermark
+          val fetchOffset = logReadResult.info.fetchOffsetMetadata.messageOffset
           if (!inSyncReplicas.contains(replica) &&
              assignedReplicas.map(_.brokerId).contains(replicaId) &&
-             replica.logEndOffset.offsetDiff(leaderHW) >= 0) {
+             replica.logEndOffset.offsetDiff(leaderHW) >= 0 &&
+             leaderEpochStartOffsetOpt.exists(fetchOffset >= _)) {
             val newInSyncReplicas = inSyncReplicas + replica
             info(s"Expanding ISR from ${inSyncReplicas.map(_.brokerId).mkString(",")} " +
               s"to ${newInSyncReplicas.map(_.brokerId).mkString(",")}")
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index 96c1147c9bf9d..343693e82c7b5 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -27,6 +27,7 @@ import kafka.common.UnexpectedAppendOffsetException
 import kafka.log.{LogConfig, LogManager, CleanerConfig}
 import kafka.server._
 import kafka.utils.{MockTime, TestUtils, MockScheduler}
+import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.ReplicaNotAvailableException
 import org.apache.kafka.common.metrics.Metrics
@@ -36,6 +37,7 @@ import org.apache.kafka.common.requests.LeaderAndIsrRequest
 import org.junit.{After, Before, Test}
 import org.junit.Assert._
 import org.scalatest.Assertions.assertThrows
+import org.easymock.EasyMock
 
 import scala.collection.JavaConverters._
 
@@ -72,10 +74,16 @@ class PartitionTest {
     val brokerProps = TestUtils.createBrokerConfig(brokerId, TestUtils.MockZkConnect)
     brokerProps.put(KafkaConfig.LogDirsProp, Seq(logDir1, logDir2).map(_.getAbsolutePath).mkString(","))
     val brokerConfig = KafkaConfig.fromProps(brokerProps)
+    val kafkaZkClient = EasyMock.createMock(classOf[KafkaZkClient])
     replicaManager = new ReplicaManager(
-      config = brokerConfig, metrics, time, zkClient = null, new MockScheduler(time),
+      config = brokerConfig, metrics, time, zkClient = kafkaZkClient, new MockScheduler(time),
       logManager, new AtomicBoolean(false), QuotaFactory.instantiate(brokerConfig, metrics, time, ""),
       brokerTopicStats, new MetadataCache(brokerId), new LogDirFailureChannel(brokerConfig.logDirs.size))
+
+    EasyMock.expect(kafkaZkClient.getEntityConfigs(EasyMock.anyString(), EasyMock.anyString())).andReturn(logProps).anyTimes()
+    EasyMock.expect(kafkaZkClient.conditionalUpdatePath(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject()))
+      .andReturn((true, 0)).anyTimes()
+    EasyMock.replay(kafkaZkClient)
   }
 
   @After
@@ -230,6 +238,82 @@ class PartitionTest {
     assertFalse(partition.makeFollower(0, partitionStateInfo, 2))
   }
 
+  @Test
+  def testFollowerDoesNotJoinISRUntilCaughtUpToOffsetWithinCurrentLeaderEpoch(): Unit = {
+    val controllerEpoch = 3
+    val leader = brokerId
+    val follower1 = brokerId + 1
+    val follower2 = brokerId + 2
+    val controllerId = brokerId + 3
+    val replicas = List[Integer](leader, follower1, follower2).asJava
+    val isr = List[Integer](leader, follower2).asJava
+    val leaderEpoch = 8
+    val batch1 = TestUtils.records(records = List(new SimpleRecord("k1".getBytes, "v1".getBytes),
+                                                  new SimpleRecord("k2".getBytes, "v2".getBytes)))
+    val batch2 = TestUtils.records(records = List(new SimpleRecord("k3".getBytes, "v1".getBytes),
+                                                  new SimpleRecord("k4".getBytes, "v2".getBytes),
+                                                  new SimpleRecord("k5".getBytes, "v3".getBytes)))
+    val batch3 = TestUtils.records(records = List(new SimpleRecord("k6".getBytes, "v1".getBytes),
+                                                  new SimpleRecord("k7".getBytes, "v2".getBytes)))
+
+    val partition = new Partition(topicPartition.topic, topicPartition.partition, time, replicaManager)
+    assertTrue("Expected first makeLeader() to return 'leader changed'",
+               partition.makeLeader(controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, leader, leaderEpoch, isr, 1, replicas, true), 0))
+    assertEquals("Current leader epoch", leaderEpoch, partition.getLeaderEpoch)
+    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas.map(_.brokerId))
+
+    // after makeLeader(() call, partition should know about all the replicas
+    val leaderReplica = partition.getReplica(leader).get
+    val follower1Replica = partition.getReplica(follower1).get
+    val follower2Replica = partition.getReplica(follower2).get
+
+    // append records with initial leader epoch
+    val lastOffsetOfFirstBatch = partition.appendRecordsToLeader(batch1, isFromClient = true).lastOffset
+    partition.appendRecordsToLeader(batch2, isFromClient = true)
+    assertEquals("Expected leader's HW not move", leaderReplica.logStartOffset, leaderReplica.highWatermark.messageOffset)
+
+    // let the follower in ISR move leader's HW to move further but below LEO
+    def readResult(fetchInfo: FetchDataInfo, leaderReplica: Replica): LogReadResult = {
+      LogReadResult(info = fetchInfo,
+                    highWatermark = leaderReplica.highWatermark.messageOffset,
+                    leaderLogStartOffset = leaderReplica.logStartOffset,
+                    leaderLogEndOffset = leaderReplica.logEndOffset.messageOffset,
+                    followerLogStartOffset = 0,
+                    fetchTimeMs = time.milliseconds,
+                    readSize = 10240,
+                    lastStableOffset = None)
+    }
+    partition.updateReplicaLogReadResult(
+      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(0), batch1), leaderReplica))
+    partition.updateReplicaLogReadResult(
+      follower2Replica, readResult(FetchDataInfo(LogOffsetMetadata(lastOffsetOfFirstBatch), batch2), leaderReplica))
+    assertEquals("Expected leader's HW", lastOffsetOfFirstBatch, leaderReplica.highWatermark.messageOffset)
+
+    // current leader becomes follower and then leader again (without any new records appended)
+    partition.makeFollower(
+      controllerId, new LeaderAndIsrRequest.PartitionState(controllerEpoch, follower2, leaderEpoch + 1, isr, 1, replicas, false), 1)
+    assertTrue("Expected makeLeader() to return 'leader changed' after makeFollower()",
+               partition.makeLeader(controllerEpoch, new LeaderAndIsrRequest.PartitionState(
+                 controllerEpoch, leader, leaderEpoch + 2, isr, 1, replicas, false), 2))
+    val currentLeaderEpochStartOffset = leaderReplica.logEndOffset.messageOffset
+
+    // append records with the latest leader epoch
+    partition.appendRecordsToLeader(batch3, isFromClient = true)
+
+    // fetch from follower not in ISR from log start offset should not add this follower to ISR
+    partition.updateReplicaLogReadResult(follower1Replica,
+                                         readResult(FetchDataInfo(LogOffsetMetadata(0), batch1), leaderReplica))
+    partition.updateReplicaLogReadResult(follower1Replica,
+                                         readResult(FetchDataInfo(LogOffsetMetadata(lastOffsetOfFirstBatch), batch2), leaderReplica))
+    assertEquals("ISR", Set[Integer](leader, follower2), partition.inSyncReplicas.map(_.brokerId))
+
+    // fetch from the follower not in ISR from start offset of the current leader epoch should
+    // add this follower to ISR
+    partition.updateReplicaLogReadResult(follower1Replica,
+                                         readResult(FetchDataInfo(LogOffsetMetadata(currentLeaderEpochStartOffset), batch3), leaderReplica))
+    assertEquals("ISR", Set[Integer](leader, follower1, follower2), partition.inSyncReplicas.map(_.brokerId))
+ }
+
   def createRecords(records: Iterable[SimpleRecord], baseOffset: Long, partitionLeaderEpoch: Int = 0): MemoryRecords = {
     val buf = ByteBuffer.allocate(DefaultRecordBatch.sizeInBytes(records.asJava))
     val builder = MemoryRecords.builder(

From fd5acd73e648a2aab4b970ddf04ad4cace6bad9a Mon Sep 17 00:00:00 2001
From: Robert Yokota 
Date: Tue, 28 Aug 2018 12:59:08 -0700
Subject: [PATCH 0756/1847] KAFKA-7242: Reverse xform configs before saving
 (KIP-297)

During actions such as a reconfiguration, the task configs are obtained
via `Worker.connectorTaskConfigs` and then subsequently saved into an
instance of `ClusterConfigState`.  The values of the properties that are saved
are post-transformation (of variable references) when they should be
pre-transformation.  This is to avoid secrets appearing in plaintext in
the `connect-configs` topic, for example.

The fix is to change the 2 clients of `Worker.connectorTaskConfigs` to
perform a reverse transformation (values converted back into variable
references) before saving them into an instance of `ClusterConfigState`.
The 2 places where the save is performed are
`DistributedHerder.reconfigureConnector` and
`StandaloneHerder.updateConnectorTasks`.

The way that the reverse transformation works is by using the
"raw" connector config (with variable references still intact) from
`ClusterConfigState` to convert config values back into variable
references for those keys that are common between the task config
and the connector config.

There are 2 additional small changes that only affect `StandaloneHerder`:

1) `ClusterConfigState.allTasksConfigs` has been changed to perform a
transformation (resolution) on all variable references.  This is
necessary because the result of this method is compared directly to
`Worker.connectorTaskConfigs`, which also has variable references
resolved.

2) `StandaloneHerder.startConnector` has been changed to match
`DistributedHerder.startConnector`.  This is to fix an issue where
during `StandaloneHerder.restartConnector`, the post-transformed
connector config would be saved back into `ClusterConfigState`.

I also performed an analysis of all other code paths where configs are
saved back into `ClusterConfigState` and did not find any other
issues.

Author: Robert Yokota 

Reviewers: Ewen Cheslack-Postava 

Closes #5475 from rayokota/KAFKA-7242-reverse-xform-props
---
 .../common/config/ConfigTransformer.java      |  2 +-
 .../kafka/connect/runtime/AbstractHerder.java | 43 ++++++++++
 .../distributed/ClusterConfigState.java       | 22 ++++-
 .../distributed/DistributedHerder.java        |  5 +-
 .../runtime/standalone/StandaloneHerder.java  | 14 ++--
 .../connect/runtime/AbstractHerderTest.java   | 81 +++++++++++++++++++
 6 files changed, 154 insertions(+), 13 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java
index f5a3737d33475..6430ffdd41915 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java
@@ -53,7 +53,7 @@
  * {@link ConfigProvider#unsubscribe(String, Set, ConfigChangeCallback)} methods.
  */
 public class ConfigTransformer {
-    private static final Pattern DEFAULT_PATTERN = Pattern.compile("\\$\\{(.*?):((.*?):)?(.*?)\\}");
+    public static final Pattern DEFAULT_PATTERN = Pattern.compile("\\$\\{(.*?):((.*?):)?(.*?)\\}");
     private static final String EMPTY_PATH = "";
 
     private final Map configProviders;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index cadb4e05d9a65..82fdeccc96ba4 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -20,9 +20,11 @@
 import org.apache.kafka.common.config.ConfigDef;
 import org.apache.kafka.common.config.ConfigDef.ConfigKey;
 import org.apache.kafka.common.config.ConfigDef.Type;
+import org.apache.kafka.common.config.ConfigTransformer;
 import org.apache.kafka.common.config.ConfigValue;
 import org.apache.kafka.connect.connector.Connector;
 import org.apache.kafka.connect.errors.NotFoundException;
+import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
 import org.apache.kafka.connect.runtime.isolation.Plugins;
 import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo;
 import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos;
@@ -46,6 +48,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
@@ -53,6 +56,8 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Abstract Herder implementation which handles connector/task lifecycle tracking. Extensions
@@ -431,4 +436,42 @@ private String trace(Throwable t) {
             return null;
         }
     }
+
+    /*
+     * Performs a reverse transformation on a set of task configs, by replacing values with variable references.
+     */
+    public static List> reverseTransform(String connName,
+                                                             ClusterConfigState configState,
+                                                             List> configs) {
+
+        // Find the config keys in the raw connector config that have variable references
+        Map rawConnConfig = configState.rawConnectorConfig(connName);
+        Set connKeysWithVariableValues = keysWithVariableValues(rawConnConfig, ConfigTransformer.DEFAULT_PATTERN);
+
+        List> result = new ArrayList<>();
+        for (Map config : configs) {
+            Map newConfig = new HashMap<>(config);
+            for (String key : connKeysWithVariableValues) {
+                if (newConfig.containsKey(key)) {
+                    newConfig.put(key, rawConnConfig.get(key));
+                }
+            }
+            result.add(newConfig);
+        }
+        return result;
+    }
+
+    private static Set keysWithVariableValues(Map rawConfig, Pattern pattern) {
+        Set keys = new HashSet<>();
+        for (Map.Entry config : rawConfig.entrySet()) {
+            if (config.getValue() != null) {
+                Matcher matcher = pattern.matcher(config.getValue());
+                if (matcher.matches()) {
+                    keys.add(config.getKey());
+                }
+            }
+        }
+        return keys;
+    }
+
 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java
index 11693b51795bc..fc6a50d2fc078 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java
@@ -123,6 +123,10 @@ public Map connectorConfig(String connector) {
         return configs;
     }
 
+    public Map rawConnectorConfig(String connector) {
+        return connectorConfigs.get(connector);
+    }
+
     /**
      * Get the target state of the connector
      * @param connector name of the connector
@@ -148,16 +152,28 @@ public Map taskConfig(ConnectorTaskId task) {
         return configs;
     }
 
+    public Map rawTaskConfig(ConnectorTaskId task) {
+        return taskConfigs.get(task);
+    }
+
     /**
-     * Get all task configs for a connector.
+     * Get all task configs for a connector.  The configurations will have been transformed by
+     * {@link org.apache.kafka.common.config.ConfigTransformer} by having all variable
+     * references replaced with the current values from external instances of
+     * {@link ConfigProvider}, and may include secrets.
      * @param connector name of the connector
      * @return a list of task configurations
      */
     public List> allTaskConfigs(String connector) {
         Map> taskConfigs = new TreeMap<>();
         for (Map.Entry> taskConfigEntry : this.taskConfigs.entrySet()) {
-            if (taskConfigEntry.getKey().connector().equals(connector))
-                taskConfigs.put(taskConfigEntry.getKey().task(), taskConfigEntry.getValue());
+            if (taskConfigEntry.getKey().connector().equals(connector)) {
+                Map configs = taskConfigEntry.getValue();
+                if (configTransformer != null) {
+                    configs = configTransformer.transform(connector, configs);
+                }
+                taskConfigs.put(taskConfigEntry.getKey().task(), configs);
+            }
         }
         return new LinkedList<>(taskConfigs.values());
     }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
index 5efb78a93e40d..f2009dbac1e47 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java
@@ -1020,8 +1020,9 @@ private void reconfigureConnector(final String connName, final Callback cb
                 }
             }
             if (changed) {
+                List> rawTaskProps = reverseTransform(connName, configState, taskProps);
                 if (isLeader()) {
-                    configBackingStore.putTaskConfigs(connName, taskProps);
+                    configBackingStore.putTaskConfigs(connName, rawTaskProps);
                     cb.onCompletion(null, null);
                 } else {
                     // We cannot forward the request on the same thread because this reconfiguration can happen as a result of connector
@@ -1031,7 +1032,7 @@ private void reconfigureConnector(final String connName, final Callback cb
                         public void run() {
                             try {
                                 String reconfigUrl = RestServer.urlJoin(leaderUrl(), "/connectors/" + connName + "/tasks");
-                                RestClient.httpRequest(reconfigUrl, "POST", taskProps, null, config);
+                                RestClient.httpRequest(reconfigUrl, "POST", rawTaskProps, null, config);
                                 cb.onCompletion(null, null);
                             } catch (ConnectException e) {
                                 log.error("Request to leader to reconfigure connector tasks failed", e);
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
index 20c6a24d3841a..40ad9803a2c0a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java
@@ -201,7 +201,9 @@ public synchronized void putConnectorConfig(String connName,
                 created = true;
             }
 
-            if (!startConnector(config)) {
+            configBackingStore.putConnectorConfig(connName, config);
+
+            if (!startConnector(connName)) {
                 callback.onCompletion(new ConnectException("Failed to start connector: " + connName), null);
                 return;
             }
@@ -270,9 +272,8 @@ public synchronized void restartConnector(String connName, Callback cb) {
         if (!configState.contains(connName))
             cb.onCompletion(new NotFoundException("Connector " + connName + " not found", null), null);
 
-        Map config = configState.connectorConfig(connName);
         worker.stopConnector(connName);
-        if (startConnector(config))
+        if (startConnector(connName))
             cb.onCompletion(null, null);
         else
             cb.onCompletion(new ConnectException("Failed to start connector: " + connName), null);
@@ -290,9 +291,7 @@ public void run() {
         return new StandaloneHerderRequest(requestSeqNum.incrementAndGet(), future);
     }
 
-    private boolean startConnector(Map connectorProps) {
-        String connName = connectorProps.get(ConnectorConfig.NAME_CONFIG);
-        configBackingStore.putConnectorConfig(connName, connectorProps);
+    private boolean startConnector(String connName) {
         Map connConfigs = configState.connectorConfig(connName);
         TargetState targetState = configState.targetState(connName);
         return worker.startConnector(connName, connConfigs, new HerderConnectorContext(this, connName), this, targetState);
@@ -336,7 +335,8 @@ private void updateConnectorTasks(String connName) {
 
         if (!newTaskConfigs.equals(oldTaskConfigs)) {
             removeConnectorTasks(connName);
-            configBackingStore.putTaskConfigs(connName, newTaskConfigs);
+            List> rawTaskConfigs = reverseTransform(connName, configState, newTaskConfigs);
+            configBackingStore.putTaskConfigs(connName, rawTaskConfigs);
             createConnectorTasks(connName, configState.targetState(connName));
         }
     }
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
index db3cf273fe737..8dbda18540171 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
@@ -20,12 +20,15 @@
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.connect.connector.ConnectRecord;
 import org.apache.kafka.connect.connector.Connector;
+import org.apache.kafka.connect.runtime.distributed.ClusterConfigState;
 import org.apache.kafka.connect.runtime.isolation.PluginDesc;
 import org.apache.kafka.connect.runtime.isolation.Plugins;
 import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
 import org.apache.kafka.connect.runtime.rest.entities.ConnectorType;
 import org.apache.kafka.connect.runtime.rest.errors.BadRequestException;
+import org.apache.kafka.connect.source.SourceConnector;
+import org.apache.kafka.connect.source.SourceTask;
 import org.apache.kafka.connect.storage.ConfigBackingStore;
 import org.apache.kafka.connect.storage.StatusBackingStore;
 import org.apache.kafka.connect.transforms.Transformation;
@@ -40,6 +43,7 @@
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -61,6 +65,53 @@
 @PrepareForTest({AbstractHerder.class})
 public class AbstractHerderTest {
 
+    private static final String CONN1 = "sourceA";
+    private static final ConnectorTaskId TASK0 = new ConnectorTaskId(CONN1, 0);
+    private static final ConnectorTaskId TASK1 = new ConnectorTaskId(CONN1, 1);
+    private static final ConnectorTaskId TASK2 = new ConnectorTaskId(CONN1, 2);
+    private static final Integer MAX_TASKS = 3;
+    private static final Map CONN1_CONFIG = new HashMap<>();
+    private static final String TEST_KEY = "testKey";
+    private static final String TEST_KEY2 = "testKey2";
+    private static final String TEST_KEY3 = "testKey3";
+    private static final String TEST_VAL = "testVal";
+    private static final String TEST_VAL2 = "testVal2";
+    private static final String TEST_REF = "${file:/tmp/somefile.txt:somevar}";
+    private static final String TEST_REF2 = "${file:/tmp/somefile2.txt:somevar2}";
+    private static final String TEST_REF3 = "${file:/tmp/somefile3.txt:somevar3}";
+    static {
+        CONN1_CONFIG.put(ConnectorConfig.NAME_CONFIG, CONN1);
+        CONN1_CONFIG.put(ConnectorConfig.TASKS_MAX_CONFIG, MAX_TASKS.toString());
+        CONN1_CONFIG.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
+        CONN1_CONFIG.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, BogusSourceConnector.class.getName());
+        CONN1_CONFIG.put(TEST_KEY, TEST_REF);
+        CONN1_CONFIG.put(TEST_KEY2, TEST_REF2);
+        CONN1_CONFIG.put(TEST_KEY3, TEST_REF3);
+    }
+    private static final Map TASK_CONFIG = new HashMap<>();
+    static {
+        TASK_CONFIG.put(TaskConfig.TASK_CLASS_CONFIG, BogusSourceTask.class.getName());
+        TASK_CONFIG.put(TEST_KEY, TEST_REF);
+    }
+    private static final List> TASK_CONFIGS = new ArrayList<>();
+    static {
+        TASK_CONFIGS.add(TASK_CONFIG);
+        TASK_CONFIGS.add(TASK_CONFIG);
+        TASK_CONFIGS.add(TASK_CONFIG);
+    }
+    private static final HashMap> TASK_CONFIGS_MAP = new HashMap<>();
+    static {
+        TASK_CONFIGS_MAP.put(TASK0, TASK_CONFIG);
+        TASK_CONFIGS_MAP.put(TASK1, TASK_CONFIG);
+        TASK_CONFIGS_MAP.put(TASK2, TASK_CONFIG);
+    }
+    private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+            Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED),
+            TASK_CONFIGS_MAP, Collections.emptySet());
+    private static final ClusterConfigState SNAPSHOT_NO_TASKS = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3),
+            Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED),
+            Collections.emptyMap(), Collections.emptySet());
+
     private final String workerId = "workerId";
     private final String kafkaClusterId = "I4ZmrWqfT2e-upky_4fdPA";
     private final int generation = 5;
@@ -248,6 +299,29 @@ public void testConfigValidationTransformsExtendResults() {
         verifyAll();
     }
 
+    @Test
+    public void testReverseTransformConfigs() throws Exception {
+        // Construct a task config with constant values for TEST_KEY and TEST_KEY2
+        Map newTaskConfig = new HashMap<>();
+        newTaskConfig.put(TaskConfig.TASK_CLASS_CONFIG, BogusSourceTask.class.getName());
+        newTaskConfig.put(TEST_KEY, TEST_VAL);
+        newTaskConfig.put(TEST_KEY2, TEST_VAL2);
+        List> newTaskConfigs = new ArrayList<>();
+        newTaskConfigs.add(newTaskConfig);
+
+        // The SNAPSHOT has a task config with TEST_KEY and TEST_REF
+        List> reverseTransformed = AbstractHerder.reverseTransform(CONN1, SNAPSHOT, newTaskConfigs);
+        assertEquals(TEST_REF, reverseTransformed.get(0).get(TEST_KEY));
+
+        // The SNAPSHOT has no task configs but does have a connector config with TEST_KEY2 and TEST_REF2
+        reverseTransformed = AbstractHerder.reverseTransform(CONN1, SNAPSHOT_NO_TASKS, newTaskConfigs);
+        assertEquals(TEST_REF2, reverseTransformed.get(0).get(TEST_KEY2));
+
+        // The reverseTransformed result should not have TEST_KEY3 since newTaskConfigs does not have TEST_KEY3
+        reverseTransformed = AbstractHerder.reverseTransform(CONN1, SNAPSHOT_NO_TASKS, newTaskConfigs);
+        assertFalse(reverseTransformed.get(0).containsKey(TEST_KEY3));
+    }
+
     private AbstractHerder createConfigValidationHerder(Class connectorClass) {
 
 
@@ -299,4 +373,11 @@ public void close() {
 
         }
     }
+
+    // We need to use a real class here due to some issue with mocking java.lang.Class
+    private abstract class BogusSourceConnector extends SourceConnector {
+    }
+
+    private abstract class BogusSourceTask extends SourceTask {
+    }
 }

From 9e23af3115ddfbf311f6b37c273327b3fa8e9a14 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Wed, 29 Aug 2018 04:17:07 +0530
Subject: [PATCH 0757/1847] KAFKA-7287: Set open ACL for old consumer znode
 path (#5503)

Reviewers: Sriharsha Chintalapani , Satish Duggana , Jun Rao 
---
 core/src/main/scala/kafka/zk/ZkData.scala            | 12 +++++++++---
 .../kafka/security/auth/ZkAuthorizationTest.scala    | 12 ++++++++++--
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index f918b616024f5..760bd67299dbf 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -429,8 +429,13 @@ object PreferredReplicaElectionZNode {
   }.map(_.toSet).getOrElse(Set.empty)
 }
 
+//old consumer path znode
+object ConsumerPathZNode {
+  def path = "/consumers"
+}
+
 object ConsumerOffset {
-  def path(group: String, topic: String, partition: Integer) = s"/consumers/${group}/offsets/${topic}/${partition}"
+  def path(group: String, topic: String, partition: Integer) = s"${ConsumerPathZNode.path}/${group}/offsets/${topic}/${partition}"
   def encode(offset: Long): Array[Byte] = offset.toString.getBytes(UTF_8)
   def decode(bytes: Array[Byte]): Option[Long] = Option(bytes).map(new String(_, UTF_8).toLong)
 }
@@ -721,7 +726,7 @@ object ZkData {
 
   // These are persistent ZK paths that should exist on kafka broker startup.
   val PersistentZkPaths = Seq(
-    "/consumers", // old consumer path
+    ConsumerPathZNode.path, // old consumer path
     BrokerIdsZNode.path,
     TopicsZNode.path,
     ConfigEntityChangeNotificationZNode.path,
@@ -743,7 +748,8 @@ object ZkData {
   }
 
   def defaultAcls(isSecure: Boolean, path: String): Seq[ACL] = {
-    if (isSecure) {
+    //Old Consumer path is kept open as different consumers will write under this node.
+    if (!ConsumerPathZNode.path.equals(path) && isSecure) {
       val acls = new ArrayBuffer[ACL]
       acls ++= ZooDefs.Ids.CREATOR_ALL_ACL.asScala
       if (!sensitivePath(path))
diff --git a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
index 19fa19dafbc27..1cdbe4b2a0ea9 100644
--- a/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
+++ b/core/src/test/scala/unit/kafka/security/auth/ZkAuthorizationTest.scala
@@ -19,10 +19,10 @@ package kafka.security.auth
 
 import kafka.admin.ZkSecurityMigrator
 import kafka.utils.{CoreUtils, Logging, TestUtils, ZkUtils}
-import kafka.zk.ZooKeeperTestHarness
+import kafka.zk.{ConsumerPathZNode, ZooKeeperTestHarness}
 import org.apache.kafka.common.KafkaException
 import org.apache.kafka.common.security.JaasUtils
-import org.apache.zookeeper.data.ACL
+import org.apache.zookeeper.data.{ACL, Stat}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
@@ -304,4 +304,12 @@ class ZkAuthorizationTest extends ZooKeeperTestHarness with Logging {
         }
     }
   }
+
+  @Test
+  def testConsumerOffsetPathAcls(): Unit = {
+    zkClient.makeSurePersistentPathExists(ConsumerPathZNode.path)
+
+    val consumerPathAcls = zkClient.currentZooKeeper.getACL(ConsumerPathZNode.path, new Stat())
+    assertTrue("old consumer znode path acls are not open", consumerPathAcls.asScala.forall(TestUtils.isAclUnsecure))
+  }
 }

From 7cef37cf55353f542db3562157547d26c992e782 Mon Sep 17 00:00:00 2001
From: Ron Dagostino 
Date: Wed, 29 Aug 2018 06:16:29 -0400
Subject: [PATCH 0758/1847] KAFKA-7324: NPE due to lack of SASLExtensions in
 SASL/OAUTHBEARER (#5552)

Set empty extensions if null is passed in.

Reviewers: Satish Duggana , Stanislav Kozlovski , Rajini Sivaram 
---
 .../common/security/auth/SaslExtensions.java  |  4 ++
 .../security/auth/SaslExtensionsCallback.java | 16 +++--
 .../OAuthBearerClientInitialResponse.java     | 63 +++++++++++++++++--
 .../OAuthBearerClientInitialResponseTest.java | 22 +++++++
 .../internals/OAuthBearerSaslClientTest.java  | 35 +++++++++--
 5 files changed, 127 insertions(+), 13 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensions.java b/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensions.java
index 75cac0533eaac..c129f1ec400f7 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensions.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensions.java
@@ -24,6 +24,10 @@
  * A simple immutable value object class holding customizable SASL extensions
  */
 public class SaslExtensions {
+    /**
+     * An "empty" instance indicating no SASL extensions
+     */
+    public static final SaslExtensions NO_SASL_EXTENSIONS = new SaslExtensions(Collections.emptyMap());
     private final Map extensionsMap;
 
     public SaslExtensions(Map extensionsMap) {
diff --git a/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensionsCallback.java b/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensionsCallback.java
index d07be320625ad..c5bd449e0cc08 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensionsCallback.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/auth/SaslExtensionsCallback.java
@@ -17,6 +17,8 @@
 
 package org.apache.kafka.common.security.auth;
 
+import java.util.Objects;
+
 import javax.security.auth.callback.Callback;
 
 /**
@@ -24,11 +26,14 @@
  * in the SASL exchange.
  */
 public class SaslExtensionsCallback implements Callback {
-    private SaslExtensions extensions;
+    private SaslExtensions extensions = SaslExtensions.NO_SASL_EXTENSIONS;
 
     /**
-     * Returns a {@link SaslExtensions} consisting of the extension names and values that are sent by the client to
-     * the server in the initial client SASL authentication message.
+     * Returns always non-null {@link SaslExtensions} consisting of the extension
+     * names and values that are sent by the client to the server in the initial
+     * client SASL authentication message. The default value is
+     * {@link SaslExtensions#NO_SASL_EXTENSIONS} so that if this callback is
+     * unhandled the client will see a non-null value.
      */
     public SaslExtensions extensions() {
         return extensions;
@@ -36,8 +41,11 @@ public SaslExtensions extensions() {
 
     /**
      * Sets the SASL extensions on this callback.
+     * 
+     * @param extensions
+     *            the mandatory extensions to set
      */
     public void extensions(SaslExtensions extensions) {
-        this.extensions = extensions;
+        this.extensions = Objects.requireNonNull(extensions, "extensions must not be null");
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponse.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponse.java
index ef16ea237d4bf..a356f0da3ddb9 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponse.java
@@ -22,6 +22,7 @@
 import javax.security.sasl.SaslException;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -58,7 +59,9 @@ public OAuthBearerClientInitialResponse(byte[] response) throws SaslException {
         if (auth == null)
             throw new SaslException("Invalid OAUTHBEARER client first message: 'auth' not specified");
         properties.remove(AUTH_KEY);
-        this.saslExtensions = validateExtensions(new SaslExtensions(properties));
+        SaslExtensions extensions = new SaslExtensions(properties);
+        validateExtensions(extensions);
+        this.saslExtensions = extensions;
 
         Matcher authMatcher = AUTH_PATTERN.matcher(auth);
         if (!authMatcher.matches())
@@ -71,16 +74,48 @@ public OAuthBearerClientInitialResponse(byte[] response) throws SaslException {
         this.tokenValue = authMatcher.group("token");
     }
 
+    /**
+     * Constructor
+     * 
+     * @param tokenValue
+     *            the mandatory token value
+     * @param extensions
+     *            the optional extensions
+     * @throws SaslException
+     *             if any extension name or value fails to conform to the required
+     *             regular expression as defined by the specification, or if the
+     *             reserved {@code auth} appears as a key
+     */
     public OAuthBearerClientInitialResponse(String tokenValue, SaslExtensions extensions) throws SaslException {
         this(tokenValue, "", extensions);
     }
 
+    /**
+     * Constructor
+     * 
+     * @param tokenValue
+     *            the mandatory token value
+     * @param authorizationId
+     *            the optional authorization ID
+     * @param extensions
+     *            the optional extensions
+     * @throws SaslException
+     *             if any extension name or value fails to conform to the required
+     *             regular expression as defined by the specification, or if the
+     *             reserved {@code auth} appears as a key
+     */
     public OAuthBearerClientInitialResponse(String tokenValue, String authorizationId, SaslExtensions extensions) throws SaslException {
-        this.tokenValue = tokenValue;
+        this.tokenValue = Objects.requireNonNull(tokenValue, "token value must not be null");
         this.authorizationId = authorizationId == null ? "" : authorizationId;
-        this.saslExtensions = validateExtensions(extensions);
+        validateExtensions(extensions);
+        this.saslExtensions = extensions != null ? extensions : SaslExtensions.NO_SASL_EXTENSIONS;
     }
 
+    /**
+     * Return the always non-null extensions
+     * 
+     * @return the always non-null extensions
+     */
     public SaslExtensions extensions() {
         return saslExtensions;
     }
@@ -97,10 +132,20 @@ public byte[] toBytes() {
         return message.getBytes(StandardCharsets.UTF_8);
     }
 
+    /**
+     * Return the always non-null token value
+     * 
+     * @return the always non-null toklen value
+     */
     public String tokenValue() {
         return tokenValue;
     }
 
+    /**
+     * Return the always non-null authorization ID
+     * 
+     * @return the always non-null authorization ID
+     */
     public String authorizationId() {
         return authorizationId;
     }
@@ -108,10 +153,19 @@ public String authorizationId() {
     /**
      * Validates that the given extensions conform to the standard. They should also not contain the reserve key name {@link OAuthBearerClientInitialResponse#AUTH_KEY}
      *
+     * @param extensions
+     *            optional extensions to validate
+     * @throws SaslException
+     *             if any extension name or value fails to conform to the required
+     *             regular expression as defined by the specification, or if the
+     *             reserved {@code auth} appears as a key
+     *
      * @see RFC 7628,
      *  Section 3.1
      */
-    public static SaslExtensions validateExtensions(SaslExtensions extensions) throws SaslException {
+    public static void validateExtensions(SaslExtensions extensions) throws SaslException {
+        if (extensions == null)
+            return;
         if (extensions.map().containsKey(OAuthBearerClientInitialResponse.AUTH_KEY))
             throw new SaslException("Extension name " + OAuthBearerClientInitialResponse.AUTH_KEY + " is invalid");
 
@@ -124,7 +178,6 @@ public static SaslExtensions validateExtensions(SaslExtensions extensions) throw
             if (!EXTENSION_VALUE_PATTERN.matcher(extensionValue).matches())
                 throw new SaslException("Extension value (" + extensionValue + ") for extension " + extensionName + " is invalid");
         }
-        return extensions;
     }
 
     /**
diff --git a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponseTest.java b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponseTest.java
index 3de6408accdd5..0ba956561dfd5 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerClientInitialResponseTest.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common.security.oauthbearer.internals;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import org.apache.kafka.common.security.auth.SaslExtensions;
 import org.junit.Test;
@@ -99,4 +100,25 @@ public void testRfc7688Example() throws Exception {
         assertEquals("server.example.com", response.extensions().map().get("host"));
         assertEquals("143", response.extensions().map().get("port"));
     }
+
+    @Test
+    public void testNoExtensionsFromByteArray() throws Exception {
+        String message = "n,a=user@example.com,\u0001" +
+                "auth=Bearer vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg\u0001\u0001";
+        OAuthBearerClientInitialResponse response = new OAuthBearerClientInitialResponse(message.getBytes(StandardCharsets.UTF_8));
+        assertEquals("vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg", response.tokenValue());
+        assertEquals("user@example.com", response.authorizationId());
+        assertTrue(response.extensions().map().isEmpty());
+    }
+
+    @Test
+    public void testNoExtensionsFromTokenAndNullExtensions() throws Exception {
+        OAuthBearerClientInitialResponse response = new OAuthBearerClientInitialResponse("token", null);
+        assertTrue(response.extensions().map().isEmpty());
+    }
+
+    @Test
+    public void testValidateNullExtensions() throws Exception {
+        OAuthBearerClientInitialResponse.validateExtensions(null);
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientTest.java b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientTest.java
index 55a86245da409..fad743136f33b 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientTest.java
@@ -20,8 +20,8 @@
 import org.apache.kafka.common.security.auth.SaslExtensionsCallback;
 import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler;
 import org.apache.kafka.common.security.auth.SaslExtensions;
+import org.apache.kafka.common.security.oauthbearer.OAuthBearerToken;
 import org.apache.kafka.common.security.oauthbearer.OAuthBearerTokenCallback;
-import org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredJws;
 import org.easymock.EasyMockSupport;
 import org.junit.Test;
 
@@ -30,9 +30,11 @@
 import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.sasl.SaslException;
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -70,7 +72,32 @@ public void configure(Map configs, String saslMechanism, List scope() {
+                            return Collections.emptySet();
+                        }
+
+                        @Override
+                        public long lifetimeMs() {
+                            return 100;
+                        }
+
+                        @Override
+                        public String principalName() {
+                            return "principalName";
+                        }
+
+                        @Override
+                        public Long startTimeMs() {
+                            return null;
+                        }
+                    });
                 else if (callback instanceof SaslExtensionsCallback) {
                     if (toThrow)
                         throw new ConfigException(errorMessage);
@@ -88,7 +115,7 @@ public void close() {
 
     @Test
     public void testAttachesExtensionsToFirstClientMessage() throws Exception {
-        String expectedToken = new String(new OAuthBearerClientInitialResponse(null, testExtensions).toBytes(), StandardCharsets.UTF_8);
+        String expectedToken = new String(new OAuthBearerClientInitialResponse("", testExtensions).toBytes(), StandardCharsets.UTF_8);
 
         OAuthBearerSaslClient client = new OAuthBearerSaslClient(new ExtensionsCallbackHandler(false));
 
@@ -101,7 +128,7 @@ public void testAttachesExtensionsToFirstClientMessage() throws Exception {
     public void testNoExtensionsDoesNotAttachAnythingToFirstClientMessage() throws Exception {
         TEST_PROPERTIES.clear();
         testExtensions = new SaslExtensions(TEST_PROPERTIES);
-        String expectedToken = new String(new OAuthBearerClientInitialResponse(null, new SaslExtensions(TEST_PROPERTIES)).toBytes(), StandardCharsets.UTF_8);
+        String expectedToken = new String(new OAuthBearerClientInitialResponse("", new SaslExtensions(TEST_PROPERTIES)).toBytes(), StandardCharsets.UTF_8);
         OAuthBearerSaslClient client = new OAuthBearerSaslClient(new ExtensionsCallbackHandler(false));
 
         String message = new String(client.evaluateChallenge("".getBytes()), StandardCharsets.UTF_8);

From c580f08b7775e4431a4a1559ef7239b69708a782 Mon Sep 17 00:00:00 2001
From: Andras Katona <41361962+akatona84@users.noreply.github.com>
Date: Wed, 29 Aug 2018 13:30:39 +0200
Subject: [PATCH 0759/1847] KAFKA-7134: KafkaLog4jAppender exception handling
 with ignoreExceptions (#5415)

Reviewers: Andras Beni , Sandor Murakozi , Rajini Sivaram 
---
 build.gradle                                  |   1 +
 .../log4jappender/KafkaLog4jAppender.java     |  28 ++++-
 .../log4jappender/KafkaLog4jAppenderTest.java | 112 ++++++++++++++++--
 .../log4jappender/MockKafkaLog4jAppender.java |   6 +-
 4 files changed, 135 insertions(+), 12 deletions(-)

diff --git a/build.gradle b/build.gradle
index 9b4610d0b2274..b13e94856dfb2 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1235,6 +1235,7 @@ project(':log4j-appender') {
 
     testCompile project(':clients').sourceSets.test.output
     testCompile libs.junit
+    testCompile libs.easymock
   }
 
   javadoc {
diff --git a/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java b/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java
index 6ddaf929a6d6f..0ae7dcada931b 100644
--- a/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java
+++ b/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java
@@ -35,6 +35,7 @@
 import static org.apache.kafka.clients.producer.ProducerConfig.COMPRESSION_TYPE_CONFIG;
 import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG;
 import static org.apache.kafka.clients.producer.ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG;
+import static org.apache.kafka.clients.producer.ProducerConfig.MAX_BLOCK_MS_CONFIG;
 import static org.apache.kafka.clients.producer.ProducerConfig.RETRIES_CONFIG;
 import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG;
 import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG;
@@ -64,10 +65,12 @@ public class KafkaLog4jAppender extends AppenderSkeleton {
     private String saslKerberosServiceName;
     private String clientJaasConfPath;
     private String kerb5ConfPath;
+    private Integer maxBlockMs;
 
     private int retries = Integer.MAX_VALUE;
     private int requiredNumAcks = 1;
     private int deliveryTimeoutMs = 120000;
+    private boolean ignoreExceptions = true;
     private boolean syncSend;
     private Producer producer;
     
@@ -123,6 +126,14 @@ public void setTopic(String topic) {
         this.topic = topic;
     }
 
+    public boolean getIgnoreExceptions() {
+        return ignoreExceptions;
+    }
+
+    public void setIgnoreExceptions(boolean ignoreExceptions) {
+        this.ignoreExceptions = ignoreExceptions;
+    }
+
     public boolean getSyncSend() {
         return syncSend;
     }
@@ -203,6 +214,14 @@ public String getKerb5ConfPath() {
         return kerb5ConfPath;
     }
 
+    public int getMaxBlockMs() {
+        return maxBlockMs;
+    }
+
+    public void setMaxBlockMs(int maxBlockMs) {
+        this.maxBlockMs = maxBlockMs;
+    }
+
     @Override
     public void activateOptions() {
         // check for config parameter validity
@@ -242,6 +261,9 @@ public void activateOptions() {
                 System.setProperty("java.security.krb5.conf", kerb5ConfPath);
             }
         }
+        if (maxBlockMs != null) {
+            props.put(MAX_BLOCK_MS_CONFIG, maxBlockMs);
+        }
 
         props.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
         props.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
@@ -259,12 +281,14 @@ protected void append(LoggingEvent event) {
         String message = subAppend(event);
         LogLog.debug("[" + new Date(event.getTimeStamp()) + "]" + message);
         Future response = producer.send(
-            new ProducerRecord(topic, message.getBytes(StandardCharsets.UTF_8)));
+            new ProducerRecord<>(topic, message.getBytes(StandardCharsets.UTF_8)));
         if (syncSend) {
             try {
                 response.get();
             } catch (InterruptedException | ExecutionException ex) {
-                throw new RuntimeException(ex);
+                if (!ignoreExceptions)
+                    throw new RuntimeException(ex);
+                LogLog.debug("Exception while getting response", ex);
             }
         }
     }
diff --git a/log4j-appender/src/test/java/org/apache/kafka/log4jappender/KafkaLog4jAppenderTest.java b/log4j-appender/src/test/java/org/apache/kafka/log4jappender/KafkaLog4jAppenderTest.java
index 34be2e93c8803..d5342e8f9e35d 100644
--- a/log4j-appender/src/test/java/org/apache/kafka/log4jappender/KafkaLog4jAppenderTest.java
+++ b/log4j-appender/src/test/java/org/apache/kafka/log4jappender/KafkaLog4jAppenderTest.java
@@ -16,18 +16,31 @@
  */
 package org.apache.kafka.log4jappender;
 
+import org.apache.kafka.clients.producer.MockProducer;
+import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.log4j.Logger;
 import org.apache.log4j.PropertyConfigurator;
+import org.apache.log4j.helpers.LogLog;
+import org.easymock.EasyMock;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
-import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
 import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
 
 public class KafkaLog4jAppenderTest {
 
-    Logger logger = Logger.getLogger(KafkaLog4jAppenderTest.class);
+    private Logger logger = Logger.getLogger(KafkaLog4jAppenderTest.class);
+
+    @Before
+    public void setup() {
+        LogLog.setInternalDebugging(true);
+    }
 
     @Test
     public void testKafkaLog4jConfigs() {
@@ -66,22 +79,103 @@ public void testKafkaLog4jConfigs() {
 
 
     @Test
-    public void testLog4jAppends() throws UnsupportedEncodingException {
-        PropertyConfigurator.configure(getLog4jConfig());
+    public void testLog4jAppends() {
+        PropertyConfigurator.configure(getLog4jConfig(false));
 
         for (int i = 1; i <= 5; ++i) {
             logger.error(getMessage(i));
         }
 
         Assert.assertEquals(
-                5, ((MockKafkaLog4jAppender) (logger.getRootLogger().getAppender("KAFKA"))).getHistory().size());
+            5, (getMockKafkaLog4jAppender()).getHistory().size());
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testLog4jAppendsWithSyncSendAndSimulateProducerFailShouldThrowException() {
+        Properties props = getLog4jConfig(true);
+        props.put("log4j.appender.KAFKA.IgnoreExceptions", "false");
+        PropertyConfigurator.configure(props);
+
+        MockKafkaLog4jAppender mockKafkaLog4jAppender = getMockKafkaLog4jAppender();
+        replaceProducerWithMocked(mockKafkaLog4jAppender, false);
+
+        logger.error(getMessage(0));
+    }
+
+    @Test
+    public void testLog4jAppendsWithSyncSendWithoutIgnoringExceptionsShouldNotThrowException() {
+        Properties props = getLog4jConfig(true);
+        props.put("log4j.appender.KAFKA.IgnoreExceptions", "false");
+        PropertyConfigurator.configure(props);
+
+        MockKafkaLog4jAppender mockKafkaLog4jAppender = getMockKafkaLog4jAppender();
+        replaceProducerWithMocked(mockKafkaLog4jAppender, true);
+
+        logger.error(getMessage(0));
+    }
+
+    @Test
+    public void testLog4jAppendsWithRealProducerConfigWithSyncSendShouldNotThrowException() {
+        Properties props = getLog4jConfigWithRealProducer(true);
+        PropertyConfigurator.configure(props);
+
+        logger.error(getMessage(0));
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testLog4jAppendsWithRealProducerConfigWithSyncSendAndNotIgnoringExceptionsShouldThrowException() {
+        Properties props = getLog4jConfigWithRealProducer(false);
+        PropertyConfigurator.configure(props);
+
+        logger.error(getMessage(0));
     }
 
-    private byte[] getMessage(int i) throws UnsupportedEncodingException {
-        return ("test_" + i).getBytes("UTF-8");
+    private void replaceProducerWithMocked(MockKafkaLog4jAppender mockKafkaLog4jAppender, boolean success) {
+        @SuppressWarnings("unchecked")
+        MockProducer producer = EasyMock.niceMock(MockProducer.class);
+        @SuppressWarnings("unchecked")
+        Future futureMock = EasyMock.niceMock(Future.class);
+        try {
+            if (!success)
+                EasyMock.expect(futureMock.get())
+                    .andThrow(new ExecutionException("simulated timeout", new TimeoutException()));
+        } catch (InterruptedException | ExecutionException e) {
+            // just mocking
+        }
+        EasyMock.expect(producer.send(EasyMock.anyObject())).andReturn(futureMock);
+        EasyMock.replay(producer, futureMock);
+        // reconfiguring mock appender
+        mockKafkaLog4jAppender.setKafkaProducer(producer);
+        mockKafkaLog4jAppender.activateOptions();
+    }
+
+    private MockKafkaLog4jAppender getMockKafkaLog4jAppender() {
+        return (MockKafkaLog4jAppender) Logger.getRootLogger().getAppender("KAFKA");
+    }
+
+    private byte[] getMessage(int i) {
+        return ("test_" + i).getBytes(StandardCharsets.UTF_8);
+    }
+
+    private Properties getLog4jConfigWithRealProducer(boolean ignoreExceptions) {
+        Properties props = new Properties();
+        props.put("log4j.rootLogger", "INFO, KAFKA");
+        props.put("log4j.appender.KAFKA", "org.apache.kafka.log4jappender.KafkaLog4jAppender");
+        props.put("log4j.appender.KAFKA.layout", "org.apache.log4j.PatternLayout");
+        props.put("log4j.appender.KAFKA.layout.ConversionPattern", "%-5p: %c - %m%n");
+        props.put("log4j.appender.KAFKA.BrokerList", "127.0.0.2:9093");
+        props.put("log4j.appender.KAFKA.Topic", "test-topic");
+        props.put("log4j.appender.KAFKA.RequiredNumAcks", "1");
+        props.put("log4j.appender.KAFKA.SyncSend", "true");
+        // setting producer timeout (max.block.ms) to be low
+        props.put("log4j.appender.KAFKA.maxBlockMs", "10");
+        // ignoring exceptions
+        props.put("log4j.appender.KAFKA.IgnoreExceptions", Boolean.toString(ignoreExceptions));
+        props.put("log4j.logger.kafka.log4j", "INFO, KAFKA");
+        return props;
     }
 
-    private Properties getLog4jConfig() {
+    private Properties getLog4jConfig(boolean syncSend) {
         Properties props = new Properties();
         props.put("log4j.rootLogger", "INFO, KAFKA");
         props.put("log4j.appender.KAFKA", "org.apache.kafka.log4jappender.MockKafkaLog4jAppender");
@@ -90,7 +184,7 @@ private Properties getLog4jConfig() {
         props.put("log4j.appender.KAFKA.BrokerList", "127.0.0.1:9093");
         props.put("log4j.appender.KAFKA.Topic", "test-topic");
         props.put("log4j.appender.KAFKA.RequiredNumAcks", "1");
-        props.put("log4j.appender.KAFKA.SyncSend", "false");
+        props.put("log4j.appender.KAFKA.SyncSend", Boolean.toString(syncSend));
         props.put("log4j.logger.kafka.log4j", "INFO, KAFKA");
         return props;
     }
diff --git a/log4j-appender/src/test/java/org/apache/kafka/log4jappender/MockKafkaLog4jAppender.java b/log4j-appender/src/test/java/org/apache/kafka/log4jappender/MockKafkaLog4jAppender.java
index 8040be4ccc68c..a9eb5fb072152 100644
--- a/log4j-appender/src/test/java/org/apache/kafka/log4jappender/MockKafkaLog4jAppender.java
+++ b/log4j-appender/src/test/java/org/apache/kafka/log4jappender/MockKafkaLog4jAppender.java
@@ -34,6 +34,10 @@ protected Producer getKafkaProducer(Properties props) {
         return mockProducer;
     }
 
+    void setKafkaProducer(MockProducer producer) {
+        this.mockProducer = producer;
+    }
+
     @Override
     protected void append(LoggingEvent event) {
         if (super.getProducer() == null) {
@@ -42,7 +46,7 @@ protected void append(LoggingEvent event) {
         super.append(event);
     }
 
-    protected List> getHistory() {
+    List> getHistory() {
         return mockProducer.history();
     }
 }

From d76965e38f6a66ab37c14014a2efea7816ab73cb Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Wed, 29 Aug 2018 07:15:08 -0700
Subject: [PATCH 0760/1847] Fix SslTransportLayerTest.testUnsupportedCiphers to
 work with Java 11 (#5570)

Java 11 supports TLS 1.3 which has different cipher names than
previous TLS versions so the simplistic way of choosing ciphers
is not guaranteed to work. Fix it by configuring the context
to use TLS 1.2.

Reviewers: Rajini Sivaram 
---
 .../apache/kafka/common/network/SslTransportLayerTest.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
index d70a448df228f..919c1675d18af 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
@@ -543,7 +543,9 @@ public void testUnsupportedTLSVersion() throws Exception {
     @Test
     public void testUnsupportedCiphers() throws Exception {
         String node = "0";
-        String[] cipherSuites = SSLContext.getDefault().getDefaultSSLParameters().getCipherSuites();
+        SSLContext context = SSLContext.getInstance("TLSv1.2");
+        context.init(null, null, null);
+        String[] cipherSuites = context.getDefaultSSLParameters().getCipherSuites();
         sslServerConfigs.put(SslConfigs.SSL_CIPHER_SUITES_CONFIG, Arrays.asList(cipherSuites[0]));
         server = createEchoServer(SecurityProtocol.SSL);
 

From d0fcf12832d9d383a5c76c174119e96e36d94eaf Mon Sep 17 00:00:00 2001
From: huxihx 
Date: Wed, 29 Aug 2018 08:29:59 -0700
Subject: [PATCH 0761/1847] KAFKA-7354; Fix IdlePercent and
 NetworkProcessorAvgIdlePercent metric

Currently, MBean `kafka.network:type=Processor,name=IdlePercent,networkProcessor=*` and `afka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent` could be greater than 1. However, these two values represent a percentage which should not exceed 1.

Author: huxihx 

Reviewers: Dong Lin 

Closes #5584 from huxihx/KAFKA-7354
---
 core/src/main/scala/kafka/network/SocketServer.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index 749c921ee02f8..96feee8900343 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -102,7 +102,7 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time
             metrics.metricName("io-wait-ratio", "socket-server-metrics", p.metricTags)
           }
           ioWaitRatioMetricNames.map { metricName =>
-            Option(metrics.metric(metricName)).fold(0.0)(_.value)
+            Option(metrics.metric(metricName)).fold(0.0)(m => Math.min(m.metricValue.asInstanceOf[Double], 1.0))
           }.sum / processors.size
         }
       }
@@ -538,7 +538,8 @@ private[kafka] class Processor(val id: Int,
   newGauge(IdlePercentMetricName,
     new Gauge[Double] {
       def value = {
-        Option(metrics.metric(metrics.metricName("io-wait-ratio", "socket-server-metrics", metricTags))).fold(0.0)(_.value)
+        Option(metrics.metric(metrics.metricName("io-wait-ratio", "socket-server-metrics", metricTags)))
+          .fold(0.0)(m => Math.min(m.metricValue.asInstanceOf[Double], 1.0))
       }
     },
     // for compatibility, only add a networkProcessor tag to the Yammer Metrics alias (the equivalent Selector metric

From 34d9ae66281602dad3a70ecd07ca0f2b6237ae8c Mon Sep 17 00:00:00 2001
From: tedyu 
Date: Wed, 29 Aug 2018 09:34:01 -0700
Subject: [PATCH 0762/1847] MINOR: Fix streams Scala peek recursive call
 (#5566)

This PR fixes the previously recursive call of Streams Scala peek

Reviewers: Joan Goyeau , Guozhang Wang , John Roesler 
---
 .../kafka/streams/scala/kstream/KStream.scala |  2 +-
 .../kafka/streams/scala/KStreamTest.scala     | 25 ++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
index 2bcbf0422aefa..8e4c9aa7eca9a 100644
--- a/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
+++ b/streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/kstream/KStream.scala
@@ -573,5 +573,5 @@ class KStream[K, V](val inner: KStreamJ[K, V]) {
    * @see `org.apache.kafka.streams.kstream.KStream#peek`
    */
   def peek(action: (K, V) => Unit): KStream[K, V] =
-    inner.peek((k: K, v: V) => action(k, v))
+    inner.peek(action.asForeachAction)
 }
diff --git a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
index e70d9007309be..3fdfee69fe3fe 100644
--- a/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
+++ b/streams/streams-scala/src/test/scala/org/apache/kafka/streams/scala/KStreamTest.scala
@@ -75,7 +75,7 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     testDriver.close()
   }
 
-  "foreach a KStream" should "side effect records" in {
+  "foreach a KStream" should "run foreach actions on records" in {
     val builder = new StreamsBuilder()
     val sourceTopic = "source"
 
@@ -85,8 +85,31 @@ class KStreamTest extends FlatSpec with Matchers with TestDriver {
     val testDriver = createTestDriver(builder)
 
     testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    acc shouldBe "value1"
+
+    testDriver.pipeRecord(sourceTopic, ("2", "value2"))
+    acc shouldBe "value1value2"
+
+    testDriver.close()
+  }
+
+  "peek a KStream" should "run peek actions on records" in {
+    val builder = new StreamsBuilder()
+    val sourceTopic = "source"
+    val sinkTopic = "sink"
+
+    var acc = ""
+    builder.stream[String, String](sourceTopic).peek((k, v) => acc += v).to(sinkTopic)
+
+    val testDriver = createTestDriver(builder)
+
+    testDriver.pipeRecord(sourceTopic, ("1", "value1"))
+    acc shouldBe "value1"
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value1"
+
     testDriver.pipeRecord(sourceTopic, ("2", "value2"))
     acc shouldBe "value1value2"
+    testDriver.readRecord[String, String](sinkTopic).value shouldBe "value2"
 
     testDriver.close()
   }

From 2514a42e84c4e4bad9ac560c09e3d397707436d8 Mon Sep 17 00:00:00 2001
From: lucapette 
Date: Wed, 29 Aug 2018 18:34:54 +0200
Subject: [PATCH 0763/1847]  KAFKA-7269: Add docs for KStream.merge (#5512)

Matthias J. Sax , Bill Bejeck 
---
 docs/streams/developer-guide/dsl-api.html | 31 ++++++++++++++++++++---
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/docs/streams/developer-guide/dsl-api.html b/docs/streams/developer-guide/dsl-api.html
index beb83a3970a27..f61f052d5d052 100644
--- a/docs/streams/developer-guide/dsl-api.html
+++ b/docs/streams/developer-guide/dsl-api.html
@@ -591,7 +591,30 @@ 
    Overview
    Peek
    
+                    
    Merge
    
+                        
      
+                            
    • KStream → KStream
      • 
+                        
    
+                    
+                        
    Merges records of two streams into one larger stream.
+                            (details)
+                            
    There is no ordering guarantee between records
+                            from different streams in the merged stream. Relative order is preserved within each input stream though (ie, records within the same input stream are processed in order)
    
+                            
    
+                              
    
+                                
    +KStream<byte[], String> stream1 = ...;
+
+KStream<byte[], String> stream2 = ...;
+
+KStream<byte[], String> merged = stream1.merge(stream2);
+                                
    
+                              
    
+                            
    
+                        
+                    
+                    
    Peek
    
                         
      
                             
    • KStream → KStream
      • 
                         
    
@@ -622,7 +645,7 @@ 
    Overview
    Print
    
+                    
    Print
    
                         
      
                             
    • KStream → void
      • 
                         
    
@@ -641,7 +664,7 @@ 
    Overview
    SelectKey
    
+                    
    SelectKey
    
                         
      
                             
    • KStream → KStream
      • 
                         
    
@@ -669,7 +692,7 @@ 
    Overview
    Table to Stream
    
+                    
    Table to Stream
    
                         
      
                             
    • KTable → KStream
      • 
                         
    

From 1918f5cb5b5ebbcfb80d19521e2ee7734346a32b Mon Sep 17 00:00:00 2001
From: Chia-Ping Tsai 
Date: Thu, 30 Aug 2018 07:01:17 +0800
Subject: [PATCH 0764/1847] MINOR: Fix swallowed NPE in KafkaServer.close()
 (#5339)

If the server fails to connect to zk, `kafkaScheduler` will be `null`
when closing  `KafkaServer`. The NPE is swallowed so it's
harmless apart from the confusing log noise.

Reviewers: Ismael Juma 
---
 core/src/main/scala/kafka/server/KafkaServer.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index c2a49a15297bc..0d13d9e0730c0 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -576,7 +576,8 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP
         if (requestHandlerPool != null)
           CoreUtils.swallow(requestHandlerPool.shutdown(), this)
 
-        CoreUtils.swallow(kafkaScheduler.shutdown(), this)
+        if (kafkaScheduler != null)
+          CoreUtils.swallow(kafkaScheduler.shutdown(), this)
 
         if (apis != null)
           CoreUtils.swallow(apis.close(), this)

From 5fb2cdb5c7c4941d3261ec92dc80ca65840de495 Mon Sep 17 00:00:00 2001
From: Bill Bejeck 
Date: Wed, 29 Aug 2018 20:41:58 -0400
Subject: [PATCH 0765/1847] MINOR:Use StoreBuilder.name() for node name (#5588)

Reviewers: Guozhang Wang , John Roesler 
---
 .../kafka/streams/kstream/internals/graph/StateStoreNode.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StateStoreNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StateStoreNode.java
index a034106912fae..6d3b8ba049e64 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StateStoreNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/StateStoreNode.java
@@ -25,7 +25,7 @@ public class StateStoreNode extends StreamsGraphNode {
     protected final StoreBuilder storeBuilder;
 
     public StateStoreNode(final StoreBuilder storeBuilder) {
-        super(storeBuilder.toString(), false);
+        super(storeBuilder.name(), false);
 
         this.storeBuilder = storeBuilder;
     }

From 5651729038406733d7ecd28430b2774473928d29 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Thu, 30 Aug 2018 16:19:34 +0100
Subject: [PATCH 0766/1847] KAFKA-7338: Specify AES128 default encryption type
 for Kerberos tests (#5586)

Java 11 enables aes128-cts-hmac-sha256-128 and aes256-cts-hmac-sha384-192 by default. These are not supported in earlier versions of Java and not supported by Apache DS libraries used by MiniKdc. To ensure that the default kerberos configuration used by Kafka integration and system tests work with all versions of Java 8 and above, configure default_tkt_enctypes and default_tgs_enctypes to use aes128-cts-hmac-sha1-96.

Reviewers: Ismael Juma 
---
 core/src/test/resources/minikdc-krb5.conf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/test/resources/minikdc-krb5.conf b/core/src/test/resources/minikdc-krb5.conf
index 0603404875525..20f1be5b1d39f 100644
--- a/core/src/test/resources/minikdc-krb5.conf
+++ b/core/src/test/resources/minikdc-krb5.conf
@@ -18,6 +18,8 @@
 [libdefaults]
 default_realm = {0}
 udp_preference_limit = 1
+default_tkt_enctypes=aes128-cts-hmac-sha1-96
+default_tgs_enctypes=aes128-cts-hmac-sha1-96
 
 [realms]
 {0} = '{'

From d31c9db4afb300aa53e0a663083f2cfd3a31c537 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Thu, 30 Aug 2018 18:49:01 +0300
Subject: [PATCH 0767/1847] MINOR: Use producer template in verifiable producer
 service (#5581)

Also remove an unused old consumer template.

Reviewers: Jason Gustafson 
---
 .../services/templates/consumer.properties    | 23 -------------------
 .../services/templates/producer.properties    |  2 +-
 .../kafkatest/services/verifiable_producer.py |  4 ++--
 3 files changed, 3 insertions(+), 26 deletions(-)
 delete mode 100644 tests/kafkatest/services/templates/consumer.properties

diff --git a/tests/kafkatest/services/templates/consumer.properties b/tests/kafkatest/services/templates/consumer.properties
deleted file mode 100644
index b8723d14fa1a8..0000000000000
--- a/tests/kafkatest/services/templates/consumer.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-# 
-#    http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# see kafka.consumer.ConsumerConfig for more details
-
-zookeeper.connect={{ zookeeper_connect }}
-zookeeper.connection.timeout.ms={{ zookeeper_connection_timeout_ms|default(6000) }}
-group.id={{ group_id|default('test-consumer-group') }}
-
-{% if consumer_timeout_ms is defined and consumer_timeout_ms is not none %}
-consumer.timeout.ms={{ consumer_timeout_ms }}
-{% endif %}
diff --git a/tests/kafkatest/services/templates/producer.properties b/tests/kafkatest/services/templates/producer.properties
index 65a48807c20c3..c41a0ea0d31ce 100644
--- a/tests/kafkatest/services/templates/producer.properties
+++ b/tests/kafkatest/services/templates/producer.properties
@@ -14,4 +14,4 @@
 # limitations under the License.
 # see kafka.producer.ProducerConfig for more details
 
-bootstrap.servers = {{ broker_list }}
+request.timeout.ms={{ request_timeout_ms }}
diff --git a/tests/kafkatest/services/verifiable_producer.py b/tests/kafkatest/services/verifiable_producer.py
index e69504890194d..7fa2654db4959 100644
--- a/tests/kafkatest/services/verifiable_producer.py
+++ b/tests/kafkatest/services/verifiable_producer.py
@@ -103,7 +103,8 @@ def java_class_name(self):
 
     def prop_file(self, node):
         idx = self.idx(node)
-        prop_file = str(self.security_config)
+        prop_file = self.render('producer.properties', request_timeout_ms=(self.request_timeout_sec * 1000))
+        prop_file += "\n{}".format(str(self.security_config))
         if self.compression_types is not None:
             compression_index = idx - 1
             self.logger.info("VerifiableProducer (index = %d) will use compression type = %s", idx,
@@ -129,7 +130,6 @@ def _worker(self, idx, node):
             self.logger.info("VerifiableProducer (index = %d) will use acks = %s", idx, self.acks)
             producer_prop_file += "\nacks=%s\n" % self.acks
 
-        producer_prop_file += "\nrequest.timeout.ms=%d\n" % (self.request_timeout_sec * 1000)
         if self.enable_idempotence:
             self.logger.info("Setting up an idempotent producer")
             producer_prop_file += "\nmax.in.flight.requests.per.connection=5\n"

From f3a4ebc8f8be12884ff67a63c7ba5ce5c5f67881 Mon Sep 17 00:00:00 2001
From: Stanislav Kozlovski 
Date: Fri, 31 Aug 2018 03:04:28 +0300
Subject: [PATCH 0768/1847] KAFKA-6859; Do not send LeaderEpochRequest for
 undefined leader epochs (#5320)

If a broker or topic has a message format < 0.11, it does not track leader epochs. LeaderEpochRequests for such will always return undefined, making the follower truncate to the highe watermark. Since there is no use to use the network for such cases, don't send a request.

Reviewers: Anna Povzner , Jason Gustafson 
---
 .../src/main/scala/kafka/log/LogSegment.scala |   2 +-
 .../server/ReplicaAlterLogDirsThread.scala    |   2 +-
 .../kafka/server/ReplicaFetcherThread.scala   |  23 ++-
 .../server/epoch/LeaderEpochFileCache.scala   |   2 +-
 .../server/ReplicaFetcherThreadTest.scala     | 152 +++++++++++++++++-
 .../kafka/server/ReplicaManagerTest.scala     |   2 +-
 6 files changed, 172 insertions(+), 11 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogSegment.scala b/core/src/main/scala/kafka/log/LogSegment.scala
index 0b7167087d16d..0c00e5588f2c4 100755
--- a/core/src/main/scala/kafka/log/LogSegment.scala
+++ b/core/src/main/scala/kafka/log/LogSegment.scala
@@ -358,7 +358,7 @@ class LogSegment private[log] (val log: FileRecords,
 
         if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) {
           leaderEpochCache.foreach { cache =>
-            if (batch.partitionLeaderEpoch > cache.latestEpoch()) // this is to avoid unnecessary warning in cache.assign()
+            if (batch.partitionLeaderEpoch > cache.latestEpoch) // this is to avoid unnecessary warning in cache.assign()
               cache.assign(batch.partitionLeaderEpoch, batch.baseOffset)
           }
           updateProducerState(producerStateManager, batch)
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 08c4a17f2d784..05dc3566c0325 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -147,7 +147,7 @@ class ReplicaAlterLogDirsThread(name: String,
 
     val (partitionsWithEpoch, partitionsWithoutEpoch) = partitionEpochOpts.partition { case (_, epochCacheOpt) => epochCacheOpt.nonEmpty }
 
-    val result = partitionsWithEpoch.map { case (tp, epochCacheOpt) => tp -> epochCacheOpt.get.latestEpoch() }
+    val result = partitionsWithEpoch.map { case (tp, epochCacheOpt) => tp -> epochCacheOpt.get.latestEpoch }
     ResultWithPartitions(result, partitionsWithoutEpoch.keys.toSet)
   }
 
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index 56335a62b6087..3b1a54f3bb184 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -95,7 +95,8 @@ class ReplicaFetcherThread(name: String,
   private val minBytes = brokerConfig.replicaFetchMinBytes
   private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes
   private val fetchSize = brokerConfig.replicaFetchMaxBytes
-  private val shouldSendLeaderEpochRequest: Boolean = brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV2
+  private val brokerSupportsLeaderEpochRequest: Boolean = brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV2
+
   private val fetchSessionHandler = new FetchSessionHandler(logContext, sourceBroker.id)
 
   private def epochCacheOpt(tp: TopicPartition): Option[LeaderEpochCache] =  replicaMgr.getReplica(tp).map(_.epochs.get)
@@ -346,19 +347,29 @@ class ReplicaFetcherThread(name: String,
     val (partitionsWithEpoch, partitionsWithoutEpoch) = partitionEpochOpts.partition { case (_, epochCacheOpt) => epochCacheOpt.nonEmpty }
 
     debug(s"Build leaderEpoch request $partitionsWithEpoch")
-    val result = partitionsWithEpoch.map { case (tp, epochCacheOpt) => tp -> epochCacheOpt.get.latestEpoch() }
+    val result = partitionsWithEpoch.map { case (tp, epochCacheOpt) => tp -> epochCacheOpt.get.latestEpoch }
     ResultWithPartitions(result, partitionsWithoutEpoch.keys.toSet)
   }
 
   override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
     var result: Map[TopicPartition, EpochEndOffset] = null
-    if (shouldSendLeaderEpochRequest) {
-      val partitionsAsJava = partitions.map { case (tp, epoch) => tp -> epoch.asInstanceOf[Integer] }.toMap.asJava
+    if (brokerSupportsLeaderEpochRequest) {
+      // skip request for partitions without epoch, as their topic log message format doesn't support epochs
+      val (partitionsWithEpoch, partitionsWithoutEpoch) = partitions.partition { case (_, epoch) => epoch != UNDEFINED_EPOCH }
+      val resultWithoutEpoch = partitionsWithoutEpoch.map { case (tp, _) => (tp, new EpochEndOffset(Errors.NONE, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)) }
+
+      if (partitionsWithEpoch.isEmpty) {
+        debug("Skipping leaderEpoch request since all partitions do not have an epoch")
+        return resultWithoutEpoch
+      }
+
+      val partitionsAsJava = partitionsWithEpoch.map { case (tp, epoch) => tp -> epoch.asInstanceOf[Integer] }.toMap.asJava
       val epochRequest = new OffsetsForLeaderEpochRequest.Builder(offsetForLeaderEpochRequestVersion, partitionsAsJava)
       try {
         val response = leaderEndpoint.sendRequest(epochRequest)
-        result = response.responseBody.asInstanceOf[OffsetsForLeaderEpochResponse].responses.asScala
-        debug(s"Receive leaderEpoch response $result")
+
+        result = response.responseBody.asInstanceOf[OffsetsForLeaderEpochResponse].responses.asScala ++ resultWithoutEpoch
+        debug(s"Receive leaderEpoch response $result; Skipped request for partitions ${partitionsWithoutEpoch.keys}")
       } catch {
         case t: Throwable =>
           warn(s"Error when sending leader epoch request for $partitions", t)
diff --git a/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala b/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala
index 23a53056f3209..88f5d6bd8e3b9 100644
--- a/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala
+++ b/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala
@@ -28,7 +28,7 @@ import scala.collection.mutable.ListBuffer
 
 trait LeaderEpochCache {
   def assign(leaderEpoch: Int, offset: Long)
-  def latestEpoch(): Int
+  def latestEpoch: Int
   def endOffsetFor(epoch: Int): (Int, Long)
   def clearAndFlushLatest(offset: Long)
   def clearAndFlushEarliest(offset: Long)
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
index fbf77404b026c..9c759cebc3f87 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
@@ -23,7 +23,9 @@ import kafka.server.QuotaFactory.UnboundedQuota
 import kafka.server.epoch.LeaderEpochCache
 import kafka.server.epoch.util.ReplicaFetcherMockBlockingSend
 import kafka.utils.TestUtils
+import org.apache.kafka.clients.ClientResponse
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.internals.PartitionStates
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.protocol.Errors._
@@ -31,7 +33,7 @@ import org.apache.kafka.common.requests.EpochEndOffset
 import org.apache.kafka.common.requests.EpochEndOffset._
 import org.apache.kafka.common.utils.SystemTime
 import org.easymock.EasyMock._
-import org.easymock.{Capture, CaptureType}
+import org.easymock.{Capture, CaptureType, IAnswer}
 import org.junit.Assert._
 import org.junit.Test
 
@@ -44,6 +46,7 @@ class ReplicaFetcherThreadTest {
   private val t1p1 = new TopicPartition("topic1", 1)
   private val t2p1 = new TopicPartition("topic2", 1)
 
+  private var toFail = false
   private val brokerEndPoint = new BrokerEndPoint(0, "localhost", 1000)
 
   @Test
@@ -71,6 +74,14 @@ class ReplicaFetcherThreadTest {
     props.put(KafkaConfig.InterBrokerProtocolVersionProp, "0.10.2")
     props.put(KafkaConfig.LogMessageFormatVersionProp, "0.10.2")
     val config = KafkaConfig.fromProps(props)
+    val leaderEndpoint = createMock(classOf[BlockingSend])
+    expect(leaderEndpoint.sendRequest(anyObject())).andAnswer(new IAnswer[ClientResponse] {
+      override def answer(): ClientResponse = {
+        toFail = true // assert no leader request is sent
+        createMock(classOf[ClientResponse])
+      }
+    })
+    replay(leaderEndpoint)
     val thread = new ReplicaFetcherThread(
       name = "bob",
       fetcherId = 0,
@@ -89,9 +100,148 @@ class ReplicaFetcherThreadTest {
       t1p1 -> new EpochEndOffset(Errors.NONE, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
     )
 
+    assertFalse("Leader request should not have been sent", toFail)
+    assertEquals("results from leader epoch request should have undefined offset", expected, result)
+  }
+
+  /**
+    * If a partition doesn't have an epoch in the cache, this means it either
+    *   does not support epochs (log message format below 0.11) or it is a bootstrapping follower.
+    *
+    * In both cases, the partition has an undefined epoch
+    *   and there is no use to send the request, as we know the broker will answer with that epoch
+    */
+  @Test
+  def shouldNotSendEpochRequestIfLastEpochUndefinedForAllPartitions(): Unit = {
+    val props = TestUtils.createBrokerConfig(1, "localhost:1234")
+    props.put(KafkaConfig.InterBrokerProtocolVersionProp, "1.0.0")
+    val config = KafkaConfig.fromProps(props)
+    val leaderEndpoint = createMock(classOf[BlockingSend])
+
+    expect(leaderEndpoint.sendRequest(anyObject())).andAnswer(new IAnswer[ClientResponse] {
+      override def answer(): ClientResponse = {
+        toFail = true // assert no leader request is sent
+        createMock(classOf[ClientResponse])
+      }
+    })
+    replay(leaderEndpoint)
+    val thread = new ReplicaFetcherThread(
+      name = "bob",
+      fetcherId = 0,
+      sourceBroker = brokerEndPoint,
+      brokerConfig = config,
+      replicaMgr = null,
+      metrics =  new Metrics(),
+      time = new SystemTime(),
+      quota = null,
+      leaderEndpointBlockingSend = Some(leaderEndpoint))
+
+
+    val result = thread.fetchEpochsFromLeader(Map(t1p0 -> UNDEFINED_EPOCH, t1p1 -> UNDEFINED_EPOCH))
+
+    val expected = Map(
+      t1p0 -> new EpochEndOffset(Errors.NONE, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET),
+      t1p1 -> new EpochEndOffset(Errors.NONE, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
+    )
+
+    assertFalse("Leader request should not have been sent", toFail)
     assertEquals("results from leader epoch request should have undefined offset", expected, result)
   }
 
+  @Test
+  def shouldFetchLeaderEpochRequestIfLastEpochDefinedForSomePartitions(): Unit = {
+    val config = KafkaConfig.fromProps(TestUtils.createBrokerConfig(1, "localhost:1234"))
+
+    //Setup all dependencies
+    val quota = createNiceMock(classOf[ReplicationQuotaManager])
+    val leaderEpochs = createNiceMock(classOf[LeaderEpochCache])
+    val logManager = createMock(classOf[LogManager])
+    val replicaAlterLogDirsManager = createMock(classOf[ReplicaAlterLogDirsManager])
+    val replica = createNiceMock(classOf[Replica])
+    val partition = createMock(classOf[Partition])
+    val replicaManager = createMock(classOf[ReplicaManager])
+
+    val leaderEpoch = 5
+
+    //Stubs
+    expect(replica.epochs).andReturn(Some(leaderEpochs)).anyTimes()
+    expect(replica.logEndOffset).andReturn(new LogOffsetMetadata(0)).anyTimes()
+    expect(replica.highWatermark).andReturn(new LogOffsetMetadata(0)).anyTimes()
+    expect(leaderEpochs.latestEpoch).andReturn(leaderEpoch).once()
+    expect(leaderEpochs.latestEpoch).andReturn(leaderEpoch).once()
+    expect(leaderEpochs.latestEpoch).andReturn(UNDEFINED_EPOCH).once()  // t2p1 doesnt support epochs
+    expect(leaderEpochs.endOffsetFor(leaderEpoch)).andReturn((leaderEpoch, 0)).anyTimes()
+    expect(replicaManager.logManager).andReturn(logManager).anyTimes()
+    expect(replicaManager.replicaAlterLogDirsManager).andReturn(replicaAlterLogDirsManager).anyTimes()
+    stub(replica, partition, replicaManager)
+
+    //Expectations
+    expect(partition.truncateTo(anyLong(), anyBoolean())).once
+
+    replay(leaderEpochs, replicaManager, logManager, quota, replica)
+
+    //Define the offsets for the OffsetsForLeaderEpochResponse
+    val offsets = Map(t1p0 -> new EpochEndOffset(leaderEpoch, 1),
+      t1p1 -> new EpochEndOffset(leaderEpoch, 1),
+      t2p1 -> new EpochEndOffset(-1, 1)).asJava
+
+    //Create the fetcher thread
+    val mockNetwork = new ReplicaFetcherMockBlockingSend(offsets, brokerEndPoint, new SystemTime())
+
+    val thread = new ReplicaFetcherThread("bob", 0, brokerEndPoint, config, replicaManager, new Metrics(), new SystemTime(), quota, Some(mockNetwork))
+
+    // topic 1 supports epoch, t2 doesn't
+    thread.addPartitions(Map(t1p0 -> 0, t1p1 -> 0, t2p1 -> 0))
+
+    assertPartitionStates(thread.partitionStates, shouldBeReadyForFetch = false, shouldBeTruncatingLog = true, shouldBeDelayed = false)
+    //Loop 1
+    thread.doWork()
+    assertEquals(1, mockNetwork.epochFetchCount)
+    assertEquals(1, mockNetwork.fetchCount)
+
+    assertPartitionStates(thread.partitionStates, shouldBeReadyForFetch = true, shouldBeTruncatingLog = false, shouldBeDelayed = false)
+
+    //Loop 2 we should not fetch epochs
+    thread.doWork()
+    assertEquals(1, mockNetwork.epochFetchCount)
+    assertEquals(2, mockNetwork.fetchCount)
+
+    assertPartitionStates(thread.partitionStates, shouldBeReadyForFetch = true, shouldBeTruncatingLog = false, shouldBeDelayed = false)
+
+    //Loop 3 we should not fetch epochs
+    thread.doWork()
+    assertEquals(1, mockNetwork.epochFetchCount)
+    assertEquals(3, mockNetwork.fetchCount)
+
+    assertPartitionStates(thread.partitionStates, shouldBeReadyForFetch = true, shouldBeTruncatingLog = false, shouldBeDelayed = false)
+
+    //Assert that truncate to is called exactly once (despite two loops)
+    verify(logManager)
+  }
+
+  /**
+    * Assert that all partitions' states are as expected
+    *
+    */
+  def assertPartitionStates(states: PartitionStates[PartitionFetchState], shouldBeReadyForFetch: Boolean,
+                            shouldBeTruncatingLog: Boolean, shouldBeDelayed: Boolean): Unit = {
+    for (tp <- List(t1p0, t1p1, t2p1)) {
+      assertEquals(
+        s"Partition $tp should${if (!shouldBeReadyForFetch) " NOT" else ""} be ready for fetching",
+        shouldBeReadyForFetch, states.stateValue(tp).isReadyForFetch)
+
+      assertEquals(
+        s"Partition $tp should${if (!shouldBeTruncatingLog) " NOT" else ""} be truncating its log",
+        shouldBeTruncatingLog,
+        states.stateValue(tp).isTruncatingLog)
+
+      assertEquals(
+        s"Partition $tp should${if (!shouldBeDelayed) " NOT" else ""} be delayed",
+        shouldBeDelayed,
+        states.stateValue(tp).isDelayed)
+    }
+  }
+
   @Test
   def shouldHandleExceptionFromBlockingSend(): Unit = {
     val props = TestUtils.createBrokerConfig(1, "localhost:1234")
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 171bcf3528c8e..41564a54b0573 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -625,7 +625,7 @@ class ReplicaManagerTest {
     val mockBrokerTopicStats = new BrokerTopicStats
     val mockLogDirFailureChannel = new LogDirFailureChannel(config.logDirs.size)
     val mockLeaderEpochCache = EasyMock.createMock(classOf[LeaderEpochCache])
-    EasyMock.expect(mockLeaderEpochCache.latestEpoch()).andReturn(leaderEpochFromLeader)
+    EasyMock.expect(mockLeaderEpochCache.latestEpoch).andReturn(leaderEpochFromLeader)
     EasyMock.expect(mockLeaderEpochCache.endOffsetFor(leaderEpochFromLeader))
       .andReturn((leaderEpochFromLeader, localLogOffset))
     EasyMock.replay(mockLeaderEpochCache)

From 5d1bfa0665ce0850ee76cee152e397c23ac329a7 Mon Sep 17 00:00:00 2001
From: Dhruvil Shah 
Date: Fri, 31 Aug 2018 04:04:33 -0700
Subject: [PATCH 0769/1847] KAFKA-6950: Delay response to failed client
 authentication to prevent potential DoS issues (KIP-306) (#5082)

Reviewers: Ron Dagostino , Ismael Juma , Rajini Sivaram 
---
 .../errors/AuthenticationException.java       |   4 +
 .../kafka/common/network/Authenticator.java   |   9 +-
 ...elayedResponseAuthenticationException.java |  27 +++
 .../kafka/common/network/KafkaChannel.java    |  27 ++-
 .../apache/kafka/common/network/Selector.java | 142 ++++++++++-
 .../SaslServerAuthenticator.java              |  31 ++-
 .../common/network/NetworkTestUtils.java      |  36 ++-
 .../kafka/common/network/NioEchoServer.java   |  12 +-
 .../common/network/SslTransportLayerTest.java |  20 +-
 .../ClientAuthenticationFailureTest.java      |   4 +-
 .../SaslAuthenticatorFailureDelayTest.java    | 229 ++++++++++++++++++
 .../authenticator/SaslAuthenticatorTest.java  |  13 +-
 .../scala/kafka/network/SocketServer.scala    |   3 +
 .../main/scala/kafka/server/KafkaConfig.scala |  12 +
 ...slGssapiSslEndToEndAuthorizationTest.scala |   4 +-
 .../integration/kafka/api/SaslSetup.scala     |   8 +-
 .../server/GssapiAuthenticationTest.scala     |  43 +++-
 .../unit/kafka/network/SocketServerTest.scala |  31 +--
 .../unit/kafka/server/KafkaConfigTest.scala   |   1 +
 .../unit/kafka/utils/JaasTestUtils.scala      |   9 +-
 20 files changed, 603 insertions(+), 62 deletions(-)
 create mode 100644 clients/src/main/java/org/apache/kafka/common/network/DelayedResponseAuthenticationException.java
 create mode 100644 clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java

diff --git a/clients/src/main/java/org/apache/kafka/common/errors/AuthenticationException.java b/clients/src/main/java/org/apache/kafka/common/errors/AuthenticationException.java
index f6458c6f22ded..7a05eba03f2bc 100644
--- a/clients/src/main/java/org/apache/kafka/common/errors/AuthenticationException.java
+++ b/clients/src/main/java/org/apache/kafka/common/errors/AuthenticationException.java
@@ -40,6 +40,10 @@ public AuthenticationException(String message) {
         super(message);
     }
 
+    public AuthenticationException(Throwable cause) {
+        super(cause);
+    }
+
     public AuthenticationException(String message, Throwable cause) {
         super(message, cause);
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/network/Authenticator.java b/clients/src/main/java/org/apache/kafka/common/network/Authenticator.java
index 4e2e7273a6815..33c2e9085168d 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Authenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Authenticator.java
@@ -37,6 +37,14 @@ public interface Authenticator extends Closeable {
      */
     void authenticate() throws AuthenticationException, IOException;
 
+    /**
+     * Perform any processing related to authentication failure. This is invoked when the channel is about to be closed
+     * because of an {@link AuthenticationException} thrown from a prior {@link #authenticate()} call.
+     * @throws IOException if read/write fails due to an I/O error
+     */
+    default void handleAuthenticationFailure() throws IOException {
+    }
+
     /**
      * Returns Principal using PrincipalBuilder
      */
@@ -46,5 +54,4 @@ public interface Authenticator extends Closeable {
      * returns true if authentication is complete otherwise returns false;
      */
     boolean complete();
-
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/network/DelayedResponseAuthenticationException.java b/clients/src/main/java/org/apache/kafka/common/network/DelayedResponseAuthenticationException.java
new file mode 100644
index 0000000000000..8474426c6095c
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/network/DelayedResponseAuthenticationException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.network;
+
+import org.apache.kafka.common.errors.AuthenticationException;
+
+public class DelayedResponseAuthenticationException extends AuthenticationException {
+    private static final long serialVersionUID = 1L;
+
+    public DelayedResponseAuthenticationException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java b/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java
index 1839729f2e79c..17dc6a33ef27b 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java
@@ -120,15 +120,22 @@ public KafkaPrincipal principal() {
      * authentication. For SASL, authentication is performed by {@link Authenticator#authenticate()}.
      */
     public void prepare() throws AuthenticationException, IOException {
+        boolean authenticating = false;
         try {
             if (!transportLayer.ready())
                 transportLayer.handshake();
-            if (transportLayer.ready() && !authenticator.complete())
+            if (transportLayer.ready() && !authenticator.complete()) {
+                authenticating = true;
                 authenticator.authenticate();
+            }
         } catch (AuthenticationException e) {
             // Clients are notified of authentication exceptions to enable operations to be terminated
             // without retries. Other errors are handled as network exceptions in Selector.
             state = new ChannelState(ChannelState.State.AUTHENTICATION_FAILED, e);
+            if (authenticating) {
+                delayCloseOnAuthenticationFailure();
+                throw new DelayedResponseAuthenticationException(e);
+            }
             throw e;
         }
         if (ready())
@@ -236,6 +243,24 @@ public ChannelMuteState muteState() {
         return muteState;
     }
 
+    /**
+     * Delay channel close on authentication failure. This will remove all read/write operations from the channel until
+     * {@link #completeCloseOnAuthenticationFailure()} is called to finish up the channel close.
+     */
+    private void delayCloseOnAuthenticationFailure() {
+        transportLayer.removeInterestOps(SelectionKey.OP_WRITE);
+    }
+
+    /**
+     * Finish up any processing on {@link #prepare()} failure.
+     * @throws IOException
+     */
+    void completeCloseOnAuthenticationFailure() throws IOException {
+        transportLayer.addInterestOps(SelectionKey.OP_WRITE);
+        // Invoke the underlying handler to finish up any processing on authentication failure
+        authenticator.handleAuthenticationFailure();
+    }
+
     /**
      * Returns true if this channel has been explicitly muted using {@link KafkaChannel#mute()}
      */
diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
index 7e32509933e55..806bda700e30d 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
@@ -86,6 +86,7 @@
 public class Selector implements Selectable, AutoCloseable {
 
     public static final long NO_IDLE_TIMEOUT_MS = -1;
+    public static final int NO_FAILED_AUTHENTICATION_DELAY = 0;
 
     private enum CloseMode {
         GRACEFUL(true),            // process outstanding staged receives, notify disconnect
@@ -119,8 +120,11 @@ private enum CloseMode {
     private final int maxReceiveSize;
     private final boolean recordTimePerConnection;
     private final IdleExpiryManager idleExpiryManager;
+    private final LinkedHashMap delayedClosingChannels;
     private final MemoryPool memoryPool;
     private final long lowMemThreshold;
+    private final int failedAuthenticationDelayMs;
+
     //indicates if the previous call to poll was able to make progress in reading already-buffered data.
     //this is used to prevent tight loops when memory is not available to read any more data
     private boolean madeReadProgressLastPoll = true;
@@ -129,6 +133,8 @@ private enum CloseMode {
      * Create a new nioSelector
      * @param maxReceiveSize Max size in bytes of a single network receive (use {@link NetworkReceive#UNLIMITED} for no limit)
      * @param connectionMaxIdleMs Max idle connection time (use {@link #NO_IDLE_TIMEOUT_MS} to disable idle timeout)
+     * @param failedAuthenticationDelayMs Minimum time by which failed authentication response and channel close should be delayed by.
+     *                                    Use {@link #NO_FAILED_AUTHENTICATION_DELAY} to disable this delay.
      * @param metrics Registry for Selector metrics
      * @param time Time implementation
      * @param metricGrpPrefix Prefix for the group of metrics registered by Selector
@@ -139,6 +145,7 @@ private enum CloseMode {
      */
     public Selector(int maxReceiveSize,
             long connectionMaxIdleMs,
+            int failedAuthenticationDelayMs,
             Metrics metrics,
             Time time,
             String metricGrpPrefix,
@@ -174,8 +181,39 @@ public Selector(int maxReceiveSize,
         this.memoryPool = memoryPool;
         this.lowMemThreshold = (long) (0.1 * this.memoryPool.size());
         this.log = logContext.logger(Selector.class);
+        this.failedAuthenticationDelayMs = failedAuthenticationDelayMs;
+        this.delayedClosingChannels = (failedAuthenticationDelayMs > NO_FAILED_AUTHENTICATION_DELAY) ? new LinkedHashMap() : null;
+    }
+
+    public Selector(int maxReceiveSize,
+                    long connectionMaxIdleMs,
+                    Metrics metrics,
+                    Time time,
+                    String metricGrpPrefix,
+                    Map metricTags,
+                    boolean metricsPerConnection,
+                    boolean recordTimePerConnection,
+                    ChannelBuilder channelBuilder,
+                    MemoryPool memoryPool,
+                    LogContext logContext) {
+        this(maxReceiveSize, connectionMaxIdleMs, NO_FAILED_AUTHENTICATION_DELAY, metrics, time, metricGrpPrefix, metricTags,
+                metricsPerConnection, recordTimePerConnection, channelBuilder, memoryPool, logContext);
     }
 
+    public Selector(int maxReceiveSize,
+                long connectionMaxIdleMs,
+                int failedAuthenticationDelayMs,
+                Metrics metrics,
+                Time time,
+                String metricGrpPrefix,
+                Map metricTags,
+                boolean metricsPerConnection,
+                ChannelBuilder channelBuilder,
+                LogContext logContext) {
+        this(maxReceiveSize, connectionMaxIdleMs, failedAuthenticationDelayMs, metrics, time, metricGrpPrefix, metricTags, metricsPerConnection, false, channelBuilder, MemoryPool.NONE, logContext);
+    }
+
+
     public Selector(int maxReceiveSize,
             long connectionMaxIdleMs,
             Metrics metrics,
@@ -185,13 +223,17 @@ public Selector(int maxReceiveSize,
             boolean metricsPerConnection,
             ChannelBuilder channelBuilder,
             LogContext logContext) {
-        this(maxReceiveSize, connectionMaxIdleMs, metrics, time, metricGrpPrefix, metricTags, metricsPerConnection, false, channelBuilder, MemoryPool.NONE, logContext);
+        this(maxReceiveSize, connectionMaxIdleMs, NO_FAILED_AUTHENTICATION_DELAY, metrics, time, metricGrpPrefix, metricTags, metricsPerConnection, channelBuilder, logContext);
     }
 
     public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, ChannelBuilder channelBuilder, LogContext logContext) {
         this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, Collections.emptyMap(), true, channelBuilder, logContext);
     }
 
+    public Selector(long connectionMaxIdleMS, int failedAuthenticationDelayMs, Metrics metrics, Time time, String metricGrpPrefix, ChannelBuilder channelBuilder, LogContext logContext) {
+        this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, failedAuthenticationDelayMs, metrics, time, metricGrpPrefix, Collections.emptyMap(), true, channelBuilder, logContext);
+    }
+
     /**
      * Begin connecting to the given address and add the connection to this nioSelector associated with the given id
      * number.
@@ -435,6 +477,9 @@ public void poll(long timeout) throws IOException {
         long endIo = time.nanoseconds();
         this.sensors.ioTime.record(endIo - endSelect, time.milliseconds());
 
+        // Close channels that were delayed and are now ready to be closed
+        completeDelayedChannelClose(endIo);
+
         // we use the time at the end of select to ensure that we don't close any connections that
         // have just been processed in pollSelectionKeys
         maybeCloseOldestConnection(endSelect);
@@ -457,15 +502,14 @@ void pollSelectionKeys(Set selectionKeys,
         for (SelectionKey key : determineHandlingOrder(selectionKeys)) {
             KafkaChannel channel = channel(key);
             long channelStartTimeNanos = recordTimePerConnection ? time.nanoseconds() : 0;
+            boolean sendFailed = false;
 
             // register all per-connection metrics at once
             sensors.maybeRegisterConnectionMetrics(channel.id());
             if (idleExpiryManager != null)
                 idleExpiryManager.update(channel.id(), currentTimeNanos);
 
-            boolean sendFailed = false;
             try {
-
                 /* complete any connections that have finished their handshake (either normally or immediately) */
                 if (isImmediatelyConnected || key.isConnectable()) {
                     if (channel.finishConnect()) {
@@ -477,8 +521,9 @@ void pollSelectionKeys(Set selectionKeys,
                                 socketChannel.socket().getSendBufferSize(),
                                 socketChannel.socket().getSoTimeout(),
                                 channel.id());
-                    } else
+                    } else {
                         continue;
+                    }
                 }
 
                 /* if channel is not ready finish prepare */
@@ -532,7 +577,11 @@ else if (e instanceof AuthenticationException) // will be logged later as error
                     log.debug("Connection with {} disconnected due to authentication exception", desc, e);
                 else
                     log.warn("Unexpected error from {}; closing connection", desc, e);
-                close(channel, sendFailed ? CloseMode.NOTIFY_ONLY : CloseMode.GRACEFUL);
+
+                if (e instanceof DelayedResponseAuthenticationException)
+                    maybeDelayCloseOnAuthenticationFailure(channel);
+                else
+                    close(channel, sendFailed ? CloseMode.NOTIFY_ONLY : CloseMode.GRACEFUL);
             } finally {
                 maybeRecordTimePerConnection(channel, channelStartTimeNanos);
             }
@@ -631,6 +680,18 @@ public void unmuteAll() {
             unmute(channel);
     }
 
+    // package-private for testing
+    void completeDelayedChannelClose(long currentTimeNanos) {
+        if (delayedClosingChannels == null)
+            return;
+
+        while (!delayedClosingChannels.isEmpty()) {
+            DelayedAuthenticationFailureClose delayedClose = delayedClosingChannels.values().iterator().next();
+            if (!delayedClose.tryClose(currentTimeNanos))
+                break;
+        }
+    }
+
     private void maybeCloseOldestConnection(long currentTimeNanos) {
         if (idleExpiryManager == null)
             return;
@@ -657,6 +718,7 @@ private void clear() {
         this.completedReceives.clear();
         this.connected.clear();
         this.disconnected.clear();
+
         // Remove closed channels after all their staged receives have been processed or if a send was requested
         for (Iterator> it = closingChannels.entrySet().iterator(); it.hasNext(); ) {
             KafkaChannel channel = it.next().getValue();
@@ -667,6 +729,7 @@ private void clear() {
                 it.remove();
             }
         }
+
         for (String channel : this.failedSends)
             this.disconnected.put(channel, ChannelState.FAILED_SEND);
         this.failedSends.clear();
@@ -707,6 +770,24 @@ public void close(String id) {
         }
     }
 
+    private void maybeDelayCloseOnAuthenticationFailure(KafkaChannel channel) {
+        DelayedAuthenticationFailureClose delayedClose = new DelayedAuthenticationFailureClose(channel, failedAuthenticationDelayMs);
+        if (delayedClosingChannels != null)
+            delayedClosingChannels.put(channel.id(), delayedClose);
+        else
+            delayedClose.closeNow();
+    }
+
+    private void handleCloseOnAuthenticationFailure(KafkaChannel channel) {
+        try {
+            channel.completeCloseOnAuthenticationFailure();
+        } catch (Exception e) {
+            log.error("Exception handling close on authentication failure node {}", channel.id(), e);
+        } finally {
+            close(channel, CloseMode.GRACEFUL);
+        }
+    }
+
     /**
      * Begin closing this connection.
      * If 'closeMode' is `CloseMode.GRACEFUL`, the channel is disconnected here, but staged receives
@@ -735,10 +816,14 @@ private void close(KafkaChannel channel, CloseMode closeMode) {
             // stagedReceives will be moved to completedReceives later along with receives from other channels
             closingChannels.put(channel.id(), channel);
             log.debug("Tracking closing connection {} to process outstanding requests", channel.id());
-        } else
+        } else {
             doClose(channel, closeMode.notifyDisconnect);
+        }
         this.channels.remove(channel.id());
 
+        if (delayedClosingChannels != null)
+            delayedClosingChannels.remove(channel.id());
+
         if (idleExpiryManager != null)
             idleExpiryManager.remove(channel.id());
     }
@@ -1064,6 +1149,46 @@ public void close() {
         }
     }
 
+    /**
+     * Encapsulate a channel that must be closed after a specific delay has elapsed due to authentication failure.
+     */
+    private class DelayedAuthenticationFailureClose {
+        private final KafkaChannel channel;
+        private final long endTimeNanos;
+        private boolean closed;
+
+        /**
+         * @param channel The channel whose close is being delayed
+         * @param delayMs The amount of time by which the operation should be delayed
+         */
+        public DelayedAuthenticationFailureClose(KafkaChannel channel, int delayMs) {
+            this.channel = channel;
+            this.endTimeNanos = time.nanoseconds() + (delayMs * 1000L * 1000L);
+            this.closed = false;
+        }
+
+        /**
+         * Try to close this channel if the delay has expired.
+         * @param currentTimeNanos The current time
+         * @return True if the delay has expired and the channel was closed; false otherwise
+         */
+        public final boolean tryClose(long currentTimeNanos) {
+            if (endTimeNanos <= currentTimeNanos)
+                closeNow();
+            return closed;
+        }
+
+        /**
+         * Close the channel now, regardless of whether the delay has expired or not.
+         */
+        public final void closeNow() {
+            if (closed)
+                throw new IllegalStateException("Attempt to close a channel that has already been closed");
+            handleCloseOnAuthenticationFailure(channel);
+            closed = true;
+        }
+    }
+
     // helper class for tracking least recently used connections to enable idle connection closing
     private static class IdleExpiryManager {
         private final Map lruConnections;
@@ -1114,4 +1239,9 @@ boolean isOutOfMemory() {
     boolean isMadeReadProgressLastPoll() {
         return madeReadProgressLastPoll;
     }
+
+    // package-private for testing
+    Map delayedClosingChannels() {
+        return delayedClosingChannels;
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
index e8f77a53e22c7..43cb0a448707b 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
@@ -118,6 +118,7 @@ private enum SaslState {
     // buffers used in `authenticate`
     private NetworkReceive netInBuffer;
     private Send netOutBuffer;
+    private Send authenticationFailureSend = null;
     // flag indicating if sasl tokens are sent as Kafka SaslAuthenticate request/responses
     private boolean enableKafkaSaslAuthenticateHeaders;
 
@@ -294,6 +295,11 @@ public boolean complete() {
         return saslState == SaslState.COMPLETE;
     }
 
+    @Override
+    public void handleAuthenticationFailure() throws IOException {
+        sendAuthenticationFailureResponse();
+    }
+
     @Override
     public void close() throws IOException {
         if (principalBuilder instanceof Closeable)
@@ -362,7 +368,7 @@ private void handleSaslToken(byte[] clientToken) throws IOException {
             RequestAndSize requestAndSize = requestContext.parseRequest(requestBuffer);
             if (apiKey != ApiKeys.SASL_AUTHENTICATE) {
                 IllegalSaslStateException e = new IllegalSaslStateException("Unexpected Kafka request of type " + apiKey + " during SASL authentication.");
-                sendKafkaResponse(requestContext, requestAndSize.request.getErrorResponse(e));
+                buildResponseOnAuthenticateFailure(requestContext, requestAndSize.request.getErrorResponse(e));
                 throw e;
             }
             if (!apiKey.isVersionSupported(version)) {
@@ -378,7 +384,8 @@ private void handleSaslToken(byte[] clientToken) throws IOException {
                 ByteBuffer responseBuf = responseToken == null ? EMPTY_BUFFER : ByteBuffer.wrap(responseToken);
                 sendKafkaResponse(requestContext, new SaslAuthenticateResponse(Errors.NONE, null, responseBuf));
             } catch (SaslAuthenticationException e) {
-                sendKafkaResponse(requestContext, new SaslAuthenticateResponse(Errors.SASL_AUTHENTICATION_FAILED, e.getMessage()));
+                buildResponseOnAuthenticateFailure(requestContext,
+                        new SaslAuthenticateResponse(Errors.SASL_AUTHENTICATION_FAILED, e.getMessage()));
                 throw e;
             } catch (SaslException e) {
                 KerberosError kerberosError = KerberosError.fromException(e);
@@ -464,7 +471,7 @@ private String handleHandshakeRequest(RequestContext context, SaslHandshakeReque
             return clientMechanism;
         } else {
             LOG.debug("SASL mechanism '{}' requested by client is not supported", clientMechanism);
-            sendKafkaResponse(context, new SaslHandshakeResponse(Errors.UNSUPPORTED_SASL_MECHANISM, enabledMechanisms));
+            buildResponseOnAuthenticateFailure(context, new SaslHandshakeResponse(Errors.UNSUPPORTED_SASL_MECHANISM, enabledMechanisms));
             throw new UnsupportedSaslMechanismException("Unsupported SASL mechanism " + clientMechanism);
         }
     }
@@ -491,6 +498,24 @@ private void handleApiVersionsRequest(RequestContext context, ApiVersionsRequest
         }
     }
 
+    /**
+     * Build a {@link Send} response on {@link #authenticate()} failure. The actual response is sent out when
+     * {@link #sendAuthenticationFailureResponse()} is called.
+     */
+    private void buildResponseOnAuthenticateFailure(RequestContext context, AbstractResponse response) {
+        authenticationFailureSend = context.buildResponse(response);
+    }
+
+    /**
+     * Send any authentication failure response that may have been previously built.
+     */
+    private void sendAuthenticationFailureResponse() throws IOException {
+        if (authenticationFailureSend == null)
+            return;
+        sendKafkaResponse(authenticationFailureSend);
+        authenticationFailureSend = null;
+    }
+
     private void sendKafkaResponse(RequestContext context, AbstractResponse response) throws IOException {
         sendKafkaResponse(context.buildResponse(response));
     }
diff --git a/clients/src/test/java/org/apache/kafka/common/network/NetworkTestUtils.java b/clients/src/test/java/org/apache/kafka/common/network/NetworkTestUtils.java
index 59980490e6807..b08c8c19a8735 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/NetworkTestUtils.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/NetworkTestUtils.java
@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -28,6 +29,7 @@
 import org.apache.kafka.common.utils.LogContext;
 import org.apache.kafka.common.security.authenticator.CredentialCache;
 import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.test.TestUtils;
 
@@ -35,16 +37,22 @@
  * Common utility functions used by transport layer and authenticator tests.
  */
 public class NetworkTestUtils {
+    public static NioEchoServer createEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol,
+                                                 AbstractConfig serverConfig, CredentialCache credentialCache, Time time) throws Exception {
+        return createEchoServer(listenerName, securityProtocol, serverConfig, credentialCache, 100, time);
+    }
 
     public static NioEchoServer createEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol,
-            AbstractConfig serverConfig, CredentialCache credentialCache) throws Exception {
-        NioEchoServer server = new NioEchoServer(listenerName, securityProtocol, serverConfig, "localhost", null, credentialCache);
+                                                 AbstractConfig serverConfig, CredentialCache credentialCache,
+                                                 int failedAuthenticationDelayMs, Time time) throws Exception {
+        NioEchoServer server = new NioEchoServer(listenerName, securityProtocol, serverConfig, "localhost",
+                null, credentialCache, failedAuthenticationDelayMs, time);
         server.start();
         return server;
     }
 
-    public static Selector createSelector(ChannelBuilder channelBuilder) {
-        return new Selector(5000, new Metrics(), new MockTime(), "MetricGroup", channelBuilder, new LogContext());
+    public static Selector createSelector(ChannelBuilder channelBuilder, Time time) {
+        return new Selector(5000, new Metrics(), time, "MetricGroup", channelBuilder, new LogContext());
     }
 
     public static void checkClientConnection(Selector selector, String node, int minMessageSize, int messageCount) throws Exception {
@@ -79,19 +87,33 @@ public static void waitForChannelReady(Selector selector, String node) throws IO
         assertTrue(selector.isChannelReady(node));
     }
 
-    public static ChannelState waitForChannelClose(Selector selector, String node, ChannelState.State channelState)
+    public static ChannelState waitForChannelClose(Selector selector, String node, ChannelState.State channelState, MockTime mockTime)
             throws IOException {
         boolean closed = false;
-        for (int i = 0; i < 30; i++) {
-            selector.poll(1000L);
+        for (int i = 0; i < 300; i++) {
+            selector.poll(100L);
             if (selector.channel(node) == null && selector.closingChannel(node) == null) {
                 closed = true;
                 break;
             }
+            if (mockTime != null)
+                mockTime.setCurrentTimeMs(mockTime.milliseconds() + 150);
         }
         assertTrue("Channel was not closed by timeout", closed);
         ChannelState finalState = selector.disconnected().get(node);
         assertEquals(channelState, finalState.state());
         return finalState;
     }
+
+    public static ChannelState waitForChannelClose(Selector selector, String node, ChannelState.State channelState) throws IOException {
+        return waitForChannelClose(selector, node, channelState, null);
+    }
+
+    public static void completeDelayedChannelClose(Selector selector, long currentTimeNanos) {
+        selector.completeDelayedChannelClose(currentTimeNanos);
+    }
+
+    public static Map delayedClosingChannels(Selector selector) {
+        return selector.delayedClosingChannels();
+    }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java
index 64b7e4e679225..bd212fdc1720c 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java
@@ -27,7 +27,7 @@
 import org.apache.kafka.common.security.scram.ScramCredential;
 import org.apache.kafka.common.security.scram.internals.ScramMechanism;
 import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.test.TestCondition;
 import org.apache.kafka.test.TestUtils;
 
@@ -69,7 +69,13 @@ public class NioEchoServer extends Thread {
     private final DelegationTokenCache tokenCache;
 
     public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol, AbstractConfig config,
-            String serverHost, ChannelBuilder channelBuilder, CredentialCache credentialCache) throws Exception {
+                         String serverHost, ChannelBuilder channelBuilder, CredentialCache credentialCache, Time time) throws Exception {
+        this(listenerName, securityProtocol, config, serverHost, channelBuilder, credentialCache, 100, time);
+    }
+
+    public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol, AbstractConfig config,
+                         String serverHost, ChannelBuilder channelBuilder, CredentialCache credentialCache,
+                         int failedAuthenticationDelayMs, Time time) throws Exception {
         super("echoserver");
         setDaemon(true);
         serverSocketChannel = ServerSocketChannel.open();
@@ -89,7 +95,7 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco
         if (channelBuilder == null)
             channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, securityProtocol, config, credentialCache, tokenCache);
         this.metrics = new Metrics();
-        this.selector = new Selector(5000, metrics, new MockTime(), "MetricGroup", channelBuilder, new LogContext());
+        this.selector = new Selector(5000, failedAuthenticationDelayMs, metrics, time, "MetricGroup", channelBuilder, new LogContext());
         acceptorThread = new AcceptorThread();
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
index 919c1675d18af..67834381764da 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
@@ -27,7 +27,6 @@
 import org.apache.kafka.common.security.TestSecurityConfig;
 import org.apache.kafka.common.security.ssl.SslFactory;
 import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
 import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.test.TestCondition;
@@ -64,6 +63,7 @@
 public class SslTransportLayerTest {
 
     private static final int BUFFER_SIZE = 4 * 1024;
+    private static Time time = Time.SYSTEM;
 
     private NioEchoServer server;
     private Selector selector;
@@ -82,7 +82,7 @@ public void setup() throws Exception {
         sslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
         this.channelBuilder = new SslChannelBuilder(Mode.CLIENT, null, false);
         this.channelBuilder.configure(sslClientConfigs);
-        this.selector = new Selector(5000, new Metrics(), new MockTime(), "MetricGroup", channelBuilder, new LogContext());
+        this.selector = new Selector(5000, new Metrics(), time, "MetricGroup", channelBuilder, new LogContext());
     }
 
     @After
@@ -204,7 +204,7 @@ protected TestSslTransportLayer newTransportLayer(String id, SelectionKey key, S
         };
         serverChannelBuilder.configure(sslServerConfigs);
         server = new NioEchoServer(ListenerName.forSecurityProtocol(SecurityProtocol.SSL), SecurityProtocol.SSL,
-                new TestSecurityConfig(sslServerConfigs), "localhost", serverChannelBuilder, null);
+                new TestSecurityConfig(sslServerConfigs), "localhost", serverChannelBuilder, null, time);
         server.start();
 
         createSelector(sslClientConfigs);
@@ -786,7 +786,7 @@ private void testIOExceptionsDuringHandshake(FailureAction readFailureAction,
             channelBuilder.flushFailureAction = flushFailureAction;
             channelBuilder.failureIndex = i;
             channelBuilder.configure(sslClientConfigs);
-            this.selector = new Selector(5000, new Metrics(), new MockTime(), "MetricGroup", channelBuilder, new LogContext());
+            this.selector = new Selector(5000, new Metrics(), time, "MetricGroup", channelBuilder, new LogContext());
 
             InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
             selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE);
@@ -829,7 +829,7 @@ public void testPeerNotifiedOfHandshakeFailure() throws Exception {
             serverChannelBuilder.flushDelayCount = i;
             server = new NioEchoServer(ListenerName.forSecurityProtocol(SecurityProtocol.SSL),
                     SecurityProtocol.SSL, new TestSecurityConfig(sslServerConfigs),
-                    "localhost", serverChannelBuilder, null);
+                    "localhost", serverChannelBuilder, null, time);
             server.start();
             createSelector(sslClientConfigs);
             InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
@@ -855,7 +855,7 @@ private void testClose(SecurityProtocol securityProtocol, ChannelBuilder clientC
         String node = "0";
         server = createEchoServer(securityProtocol);
         clientChannelBuilder.configure(sslClientConfigs);
-        this.selector = new Selector(5000, new Metrics(), new MockTime(), "MetricGroup", clientChannelBuilder, new LogContext());
+        this.selector = new Selector(5000, new Metrics(), time, "MetricGroup", clientChannelBuilder, new LogContext());
         InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
         selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE);
 
@@ -895,7 +895,7 @@ public void testServerKeystoreDynamicUpdate() throws Exception {
         ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName,
                 false, securityProtocol, config, null, null);
         server = new NioEchoServer(listenerName, securityProtocol, config,
-                "localhost", serverChannelBuilder, null);
+                "localhost", serverChannelBuilder, null, time);
         server.start();
         InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
 
@@ -955,7 +955,7 @@ public void testServerTruststoreDynamicUpdate() throws Exception {
         ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName,
                 false, securityProtocol, config, null, null);
         server = new NioEchoServer(listenerName, securityProtocol, config,
-                "localhost", serverChannelBuilder, null);
+                "localhost", serverChannelBuilder, null, time);
         server.start();
         InetSocketAddress addr = new InetSocketAddress("localhost", server.port());
 
@@ -1027,12 +1027,12 @@ private Selector createSelector(Map sslClientConfigs, final Inte
         channelBuilder.configureBufferSizes(netReadBufSize, netWriteBufSize, appBufSize);
         this.channelBuilder = channelBuilder;
         this.channelBuilder.configure(sslClientConfigs);
-        this.selector = new Selector(5000, new Metrics(), new MockTime(), "MetricGroup", channelBuilder, new LogContext());
+        this.selector = new Selector(5000, new Metrics(), time, "MetricGroup", channelBuilder, new LogContext());
         return selector;
     }
 
     private NioEchoServer createEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol) throws Exception {
-        return NetworkTestUtils.createEchoServer(listenerName, securityProtocol, new TestSecurityConfig(sslServerConfigs), null);
+        return NetworkTestUtils.createEchoServer(listenerName, securityProtocol, new TestSecurityConfig(sslServerConfigs), null, time);
     }
 
     private NioEchoServer createEchoServer(SecurityProtocol securityProtocol) throws Exception {
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java
index 413997f29316b..2fce4c5637305 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/ClientAuthenticationFailureTest.java
@@ -34,6 +34,7 @@
 import org.apache.kafka.common.security.auth.SecurityProtocol;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.MockTime;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -48,6 +49,7 @@
 import static org.junit.Assert.fail;
 
 public class ClientAuthenticationFailureTest {
+    private static MockTime time = new MockTime(50);
 
     private NioEchoServer server;
     private Map saslServerConfigs;
@@ -147,6 +149,6 @@ private NioEchoServer createEchoServer(SecurityProtocol securityProtocol) throws
 
     private NioEchoServer createEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol) throws Exception {
         return NetworkTestUtils.createEchoServer(listenerName, securityProtocol,
-                new TestSecurityConfig(saslServerConfigs), new CredentialCache());
+                new TestSecurityConfig(saslServerConfigs), new CredentialCache(), time);
     }
 }
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java
new file mode 100644
index 0000000000000..b0dfc7a123b18
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.security.authenticator;
+
+import org.apache.kafka.common.config.SaslConfigs;
+import org.apache.kafka.common.config.internals.BrokerSecurityConfigs;
+import org.apache.kafka.common.errors.SaslAuthenticationException;
+import org.apache.kafka.common.network.CertStores;
+import org.apache.kafka.common.network.ChannelBuilder;
+import org.apache.kafka.common.network.ChannelBuilders;
+import org.apache.kafka.common.network.ChannelState;
+import org.apache.kafka.common.network.ListenerName;
+import org.apache.kafka.common.network.NetworkTestUtils;
+import org.apache.kafka.common.network.NioEchoServer;
+import org.apache.kafka.common.network.Selector;
+import org.apache.kafka.common.security.JaasContext;
+import org.apache.kafka.common.security.TestSecurityConfig;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = Parameterized.class)
+public class SaslAuthenticatorFailureDelayTest {
+    private static final int BUFFER_SIZE = 4 * 1024;
+    private static MockTime time = new MockTime(50);
+
+    private NioEchoServer server;
+    private Selector selector;
+    private ChannelBuilder channelBuilder;
+    private CertStores serverCertStores;
+    private CertStores clientCertStores;
+    private Map saslClientConfigs;
+    private Map saslServerConfigs;
+    private CredentialCache credentialCache;
+    private long startTimeMs;
+    private final int failedAuthenticationDelayMs;
+
+    public SaslAuthenticatorFailureDelayTest(int failedAuthenticationDelayMs) {
+        this.failedAuthenticationDelayMs = failedAuthenticationDelayMs;
+    }
+
+    @Parameterized.Parameters(name = "failedAuthenticationDelayMs={0}")
+    public static Collection data() {
+        List values = new ArrayList<>();
+        values.add(new Object[]{0});
+        values.add(new Object[]{200});
+        return values;
+    }
+
+    @Before
+    public void setup() throws Exception {
+        LoginManager.closeAll();
+        serverCertStores = new CertStores(true, "localhost");
+        clientCertStores = new CertStores(false, "localhost");
+        saslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores);
+        saslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores);
+        credentialCache = new CredentialCache();
+        SaslAuthenticatorTest.TestLogin.loginCount.set(0);
+        startTimeMs = time.milliseconds();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        long now = time.milliseconds();
+        if (server != null)
+            this.server.close();
+        if (selector != null)
+            this.selector.close();
+        if (failedAuthenticationDelayMs != -1)
+            assertTrue("timeSpent: " + (now - startTimeMs), now - startTimeMs >= failedAuthenticationDelayMs);
+    }
+
+    /**
+     * Tests that SASL/PLAIN clients with invalid password fail authentication.
+     */
+    @Test
+    public void testInvalidPasswordSaslPlain() throws Exception {
+        String node = "0";
+        SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL;
+        TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Arrays.asList("PLAIN"));
+        jaasConfig.setClientOptions("PLAIN", TestJaasConfig.USERNAME, "invalidpassword");
+
+        server = createEchoServer(securityProtocol);
+        createAndCheckClientAuthenticationFailure(securityProtocol, node, "PLAIN",
+                "Authentication failed: Invalid username or password");
+        server.verifyAuthenticationMetrics(0, 1);
+    }
+
+    /**
+     * Tests client connection close before response for authentication failure is sent.
+     */
+    @Test
+    public void testClientConnectionClose() throws Exception {
+        String node = "0";
+        SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL;
+        TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Arrays.asList("PLAIN"));
+        jaasConfig.setClientOptions("PLAIN", TestJaasConfig.USERNAME, "invalidpassword");
+
+        server = createEchoServer(securityProtocol);
+        createClientConnection(securityProtocol, node);
+
+        Map delayedClosingChannels = NetworkTestUtils.delayedClosingChannels(server.selector());
+
+        // Wait until server has established connection with client and has processed the auth failure
+        TestUtils.waitForCondition(() -> {
+            poll(selector);
+            return !server.selector().channels().isEmpty();
+        }, "Timeout waiting for connection");
+        TestUtils.waitForCondition(() -> {
+            poll(selector);
+            return failedAuthenticationDelayMs == 0 || !delayedClosingChannels.isEmpty();
+        }, "Timeout waiting for auth failure");
+
+        selector.close();
+        selector = null;
+
+        // Now that client connection is closed, wait until server notices the disconnection and removes it from the
+        // list of connected channels and from delayed response for auth failure
+        TestUtils.waitForCondition(() -> failedAuthenticationDelayMs == 0 || delayedClosingChannels.isEmpty(),
+                "Timeout waiting for delayed response remove");
+        TestUtils.waitForCondition(() -> server.selector().channels().isEmpty(),
+                "Timeout waiting for connection close");
+
+        // Try forcing completion of delayed channel close
+        TestUtils.waitForCondition(() -> time.milliseconds() > startTimeMs + failedAuthenticationDelayMs + 1,
+                "Timeout when waiting for auth failure response timeout to elapse");
+        NetworkTestUtils.completeDelayedChannelClose(server.selector(), time.nanoseconds());
+    }
+
+    private void poll(Selector selector) {
+        try {
+            selector.poll(50);
+        } catch (IOException e) {
+            Assert.fail("Caught unexpected exception " + e);
+        }
+    }
+
+    private TestJaasConfig configureMechanisms(String clientMechanism, List serverMechanisms) {
+        saslClientConfigs.put(SaslConfigs.SASL_MECHANISM, clientMechanism);
+        saslServerConfigs.put(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG, serverMechanisms);
+        if (serverMechanisms.contains("DIGEST-MD5")) {
+            saslServerConfigs.put("digest-md5." + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS,
+                    TestDigestLoginModule.DigestServerCallbackHandler.class.getName());
+        }
+        return TestJaasConfig.createConfiguration(clientMechanism, serverMechanisms);
+    }
+
+    private void createSelector(SecurityProtocol securityProtocol, Map clientConfigs) {
+        if (selector != null) {
+            selector.close();
+            selector = null;
+        }
+
+        String saslMechanism = (String) saslClientConfigs.get(SaslConfigs.SASL_MECHANISM);
+        this.channelBuilder = ChannelBuilders.clientChannelBuilder(securityProtocol, JaasContext.Type.CLIENT,
+                new TestSecurityConfig(clientConfigs), null, saslMechanism, true);
+        this.selector = NetworkTestUtils.createSelector(channelBuilder, time);
+    }
+
+    private NioEchoServer createEchoServer(SecurityProtocol securityProtocol) throws Exception {
+        return createEchoServer(ListenerName.forSecurityProtocol(securityProtocol), securityProtocol);
+    }
+
+    private NioEchoServer createEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol) throws Exception {
+        if (failedAuthenticationDelayMs != -1)
+            return NetworkTestUtils.createEchoServer(listenerName, securityProtocol,
+                    new TestSecurityConfig(saslServerConfigs), credentialCache, failedAuthenticationDelayMs, time);
+        else
+            return NetworkTestUtils.createEchoServer(listenerName, securityProtocol,
+                    new TestSecurityConfig(saslServerConfigs), credentialCache, time);
+    }
+
+    private void createClientConnection(SecurityProtocol securityProtocol, String node) throws Exception {
+        createSelector(securityProtocol, saslClientConfigs);
+        InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port());
+        selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE);
+    }
+
+    private void createAndCheckClientAuthenticationFailure(SecurityProtocol securityProtocol, String node,
+                                                           String mechanism, String expectedErrorMessage) throws Exception {
+        ChannelState finalState = createAndCheckClientConnectionFailure(securityProtocol, node);
+        Exception exception = finalState.exception();
+        assertTrue("Invalid exception class " + exception.getClass(), exception instanceof SaslAuthenticationException);
+        if (expectedErrorMessage == null)
+            expectedErrorMessage = "Authentication failed due to invalid credentials with SASL mechanism " + mechanism;
+        assertEquals(expectedErrorMessage, exception.getMessage());
+    }
+
+    private ChannelState createAndCheckClientConnectionFailure(SecurityProtocol securityProtocol, String node)
+            throws Exception {
+        createClientConnection(securityProtocol, node);
+        ChannelState finalState = NetworkTestUtils.waitForChannelClose(selector, node,
+                ChannelState.State.AUTHENTICATION_FAILED, time);
+        selector.close();
+        selector = null;
+        return finalState;
+    }
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index b8894f1370fbd..74058eb1cd578 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -92,6 +92,7 @@
 import org.apache.kafka.common.security.authenticator.TestDigestLoginModule.DigestServerCallbackHandler;
 import org.apache.kafka.common.security.plain.internals.PlainServerCallbackHandler;
 
+import org.apache.kafka.common.utils.Time;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -107,6 +108,7 @@
 public class SaslAuthenticatorTest {
 
     private static final int BUFFER_SIZE = 4 * 1024;
+    private static Time time = Time.SYSTEM;
 
     private NioEchoServer server;
     private Selector selector;
@@ -1308,7 +1310,7 @@ protected void enableKafkaSaslAuthenticateHeaders(boolean flag) {
         };
         serverChannelBuilder.configure(saslServerConfigs);
         server = new NioEchoServer(listenerName, securityProtocol, new TestSecurityConfig(saslServerConfigs),
-                "localhost", serverChannelBuilder, credentialCache);
+                "localhost", serverChannelBuilder, credentialCache, time);
         server.start();
         return server;
     }
@@ -1347,7 +1349,7 @@ protected void saslAuthenticateVersion(short version) {
             }
         };
         clientChannelBuilder.configure(saslClientConfigs);
-        this.selector = NetworkTestUtils.createSelector(clientChannelBuilder);
+        this.selector = NetworkTestUtils.createSelector(clientChannelBuilder, time);
         InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port());
         selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE);
     }
@@ -1452,7 +1454,7 @@ private void createSelector(SecurityProtocol securityProtocol, Map (k, v.toInt)}
   val connectionsMaxIdleMs = getLong(KafkaConfig.ConnectionsMaxIdleMsProp)
+  val failedAuthenticationDelayMs = getInt(KafkaConfig.FailedAuthenticationDelayMsProp)
 
   /***************** rack configuration **************/
   val rack = Option(getString(KafkaConfig.RackProp))
@@ -1397,5 +1403,11 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO
     val invalidAddresses = maxConnectionsPerIpOverrides.keys.filterNot(address => Utils.validHostPattern(address))
     if (!invalidAddresses.isEmpty)
       throw new IllegalArgumentException(s"${KafkaConfig.MaxConnectionsPerIpOverridesProp} contains invalid addresses : ${invalidAddresses.mkString(",")}")
+
+    if (connectionsMaxIdleMs >= 0)
+      require(failedAuthenticationDelayMs < connectionsMaxIdleMs,
+        s"${KafkaConfig.FailedAuthenticationDelayMsProp}=$failedAuthenticationDelayMs should always be less than" +
+          s" ${KafkaConfig.ConnectionsMaxIdleMsProp}=$connectionsMaxIdleMs to prevent failed" +
+          " authentication responses from timing out")
   }
 }
diff --git a/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala
index a9b4a60ac7268..a630293bb6625 100644
--- a/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslGssapiSslEndToEndAuthorizationTest.scala
@@ -24,10 +24,10 @@ import scala.collection.immutable.List
 class SaslGssapiSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTest {
   override val clientPrincipal = JaasTestUtils.KafkaClientPrincipalUnqualifiedName
   override val kafkaPrincipal = JaasTestUtils.KafkaServerPrincipalUnqualifiedName
-  
+
   override protected def kafkaClientSaslMechanism = "GSSAPI"
   override protected def kafkaServerSaslMechanisms = List("GSSAPI")
-  
+
   // Configure brokers to require SSL client authentication in order to verify that SASL_SSL works correctly even if the
   // client doesn't have a keystore. We want to cover the scenario where a broker requires either SSL client
   // authentication or SASL authentication with SSL as the transport layer (but not both).
diff --git a/core/src/test/scala/integration/kafka/api/SaslSetup.scala b/core/src/test/scala/integration/kafka/api/SaslSetup.scala
index 391321227bd07..81de1059068a8 100644
--- a/core/src/test/scala/integration/kafka/api/SaslSetup.scala
+++ b/core/src/test/scala/integration/kafka/api/SaslSetup.scala
@@ -138,8 +138,12 @@ trait SaslSetup {
     props
   }
 
-  def jaasClientLoginModule(clientSaslMechanism: String): String =
-    JaasTestUtils.clientLoginModule(clientSaslMechanism, clientKeytabFile)
+  def jaasClientLoginModule(clientSaslMechanism: String, serviceName: Option[String] = None): String = {
+    if (serviceName.isDefined)
+      JaasTestUtils.clientLoginModule(clientSaslMechanism, clientKeytabFile, serviceName.get)
+    else
+      JaasTestUtils.clientLoginModule(clientSaslMechanism, clientKeytabFile)
+  }
 
   def createScramCredentials(zkConnect: String, userName: String, password: String): Unit = {
     val credentials = ScramMechanism.values.map(m => s"${m.mechanismName}=[iterations=4096,password=$password]")
diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
index 74b2a152e231e..cbe8462b3fddd 100644
--- a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
@@ -19,19 +19,25 @@
 package kafka.server
 
 import java.net.InetSocketAddress
+import java.time.Duration
 import java.util.Properties
 import java.util.concurrent.{Executors, TimeUnit}
 
 import kafka.api.{Both, IntegrationTestHarness, SaslSetup}
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.config.SaslConfigs
+import org.apache.kafka.common.errors.SaslAuthenticationException
 import org.apache.kafka.common.network._
 import org.apache.kafka.common.security.{JaasContext, TestSecurityConfig}
 import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.apache.kafka.common.utils.MockTime
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 
+import scala.collection.JavaConverters._
+
 class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
   override val serverCount = 1
   override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT
@@ -43,10 +49,17 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
   private val executor = Executors.newFixedThreadPool(numThreads)
   private val clientConfig: Properties = new Properties
   private var serverAddr: InetSocketAddress = _
+  private val time = new MockTime(10)
+  val topic = "topic"
+  val part = 0
+  val tp = new TopicPartition(topic, part)
+  private val failedAuthenticationDelayMs = 2000
 
   @Before
   override def setUp() {
     startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))
+    serverConfig.put(KafkaConfig.SslClientAuthProp, "required")
+    serverConfig.put(KafkaConfig.FailedAuthenticationDelayMsProp, failedAuthenticationDelayMs.toString)
     super.setUp()
     serverAddr = new InetSocketAddress("localhost",
       servers.head.boundPort(ListenerName.forSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT)))
@@ -55,6 +68,9 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
     clientConfig.put(SaslConfigs.SASL_MECHANISM, kafkaClientSaslMechanism)
     clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, jaasClientLoginModule(kafkaClientSaslMechanism))
     clientConfig.put(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG, "5000")
+
+    // create the test topic with all the brokers as replicas
+    createTopic(topic, 2, serverCount)
   }
 
   @After
@@ -93,6 +109,31 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
     verifyNonRetriableAuthenticationFailure()
   }
 
+  /**
+   * Test that when client fails to verify authenticity of the server, the resulting failed authentication exception
+   * is thrown immediately, and is not affected by connection.failed.authentication.delay.ms.
+   */
+  @Test
+  def testServerAuthenticationFailure(): Unit = {
+    // Setup client with a non-existent service principal, so that server authentication fails on the client
+    val clientLoginContext = jaasClientLoginModule(kafkaClientSaslMechanism, Some("another-kafka-service"))
+    val configOverrides = new Properties()
+    configOverrides.setProperty(SaslConfigs.SASL_JAAS_CONFIG, clientLoginContext)
+    val consumer = createConsumer(configOverrides = configOverrides)
+    consumer.assign(List(tp).asJava)
+
+    val startMs = System.currentTimeMillis()
+    try {
+      consumer.poll(Duration.ofMillis(50))
+      fail()
+    } catch {
+      case _: SaslAuthenticationException =>
+    }
+    val endMs = System.currentTimeMillis()
+    require(endMs - startMs < failedAuthenticationDelayMs, "Failed authentication must not be delayed on the client")
+    consumer.close()
+  }
+
   /**
    * Verifies that any exceptions during authentication with the current `clientConfig` are
    * notified with disconnect state `AUTHENTICATE` (and not `AUTHENTICATION_FAILED`). This
@@ -148,6 +189,6 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
   private def createSelector(): Selector = {
     val channelBuilder = ChannelBuilders.clientChannelBuilder(securityProtocol,
       JaasContext.Type.CLIENT, new TestSecurityConfig(clientConfig), null, kafkaClientSaslMechanism, true)
-    NetworkTestUtils.createSelector(channelBuilder)
+    NetworkTestUtils.createSelector(channelBuilder, time)
   }
 }
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index da6114907408f..b5983377b7730 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -307,13 +307,14 @@ class SocketServerTest extends JUnitSuite {
       override def newProcessor(id: Int, connectionQuotas: ConnectionQuotas, listenerName: ListenerName,
                                 protocol: SecurityProtocol, memoryPool: MemoryPool): Processor = {
         new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas,
-          config.connectionsMaxIdleMs, listenerName, protocol, config, metrics, credentialProvider, memoryPool, new LogContext()) {
-          override protected[network] def connectionId(socket: Socket): String = overrideConnectionId
-          override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = {
-           val testableSelector = new TestableSelector(config, channelBuilder, time, metrics)
-           selector = testableSelector
-           testableSelector
-        }
+          config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics,
+          credentialProvider, memoryPool, new LogContext()) {
+            override protected[network] def connectionId(socket: Socket): String = overrideConnectionId
+            override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = {
+             val testableSelector = new TestableSelector(config, channelBuilder, time, metrics)
+             selector = testableSelector
+             testableSelector
+          }
         }
       }
     }
@@ -652,7 +653,8 @@ class SocketServerTest extends JUnitSuite {
       override def newProcessor(id: Int, connectionQuotas: ConnectionQuotas, listenerName: ListenerName,
                                 protocol: SecurityProtocol, memoryPool: MemoryPool): Processor = {
         new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas,
-          config.connectionsMaxIdleMs, listenerName, protocol, config, metrics, credentialProvider, MemoryPool.NONE, new LogContext()) {
+          config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics,
+          credentialProvider, MemoryPool.NONE, new LogContext()) {
           override protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send) {
             conn.close()
             super.sendResponse(response, responseSend)
@@ -697,7 +699,8 @@ class SocketServerTest extends JUnitSuite {
       override def newProcessor(id: Int, connectionQuotas: ConnectionQuotas, listenerName: ListenerName,
                                 protocol: SecurityProtocol, memoryPool: MemoryPool): Processor = {
         new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas,
-          config.connectionsMaxIdleMs, listenerName, protocol, config, metrics, credentialProvider, memoryPool, new LogContext()) {
+          config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics,
+          credentialProvider, memoryPool, new LogContext()) {
           override protected[network] def connectionId(socket: Socket): String = overrideConnectionId
           override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = {
            val testableSelector = new TestableSelector(config, channelBuilder, time, metrics)
@@ -743,7 +746,7 @@ class SocketServerTest extends JUnitSuite {
   @Test
   def testBrokerSendAfterChannelClosedUpdatesRequestMetrics() {
     val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0)
-    props.setProperty(KafkaConfig.ConnectionsMaxIdleMsProp, "100")
+    props.setProperty(KafkaConfig.ConnectionsMaxIdleMsProp, "110")
     val serverMetrics = new Metrics
     var conn: Socket = null
     val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider)
@@ -1100,10 +1103,8 @@ class SocketServerTest extends JUnitSuite {
 
     override def newProcessor(id: Int, connectionQuotas: ConnectionQuotas, listenerName: ListenerName,
                                 protocol: SecurityProtocol, memoryPool: MemoryPool): Processor = {
-      new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas,
-        config.connectionsMaxIdleMs, listenerName, protocol, config, metrics, credentialProvider, memoryPool, new
-            LogContext()) {
-
+      new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas, config.connectionsMaxIdleMs,
+        config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, credentialProvider, memoryPool, new LogContext()) {
         override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = {
            val testableSelector = new TestableSelector(config, channelBuilder, time, metrics)
            assertEquals(None, selector)
@@ -1149,7 +1150,7 @@ class SocketServerTest extends JUnitSuite {
   }
 
   class TestableSelector(config: KafkaConfig, channelBuilder: ChannelBuilder, time: Time, metrics: Metrics)
-        extends Selector(config.socketRequestMaxBytes, config.connectionsMaxIdleMs,
+        extends Selector(config.socketRequestMaxBytes, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs,
             metrics, time, "socket-server", new HashMap, false, true, channelBuilder, MemoryPool.NONE, new LogContext()) {
 
     val failures = mutable.Map[SelectorOperation, Exception]()
diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
index 927dd1c203b17..b7a8951ecc043 100755
--- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala
@@ -587,6 +587,7 @@ class KafkaConfigTest {
         case KafkaConfig.MaxConnectionsPerIpOverridesProp =>
           assertPropertyInvalid(getBaseProperties(), name, "127.0.0.1:not_a_number")
         case KafkaConfig.ConnectionsMaxIdleMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number")
+        case KafkaConfig.FailedAuthenticationDelayMsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-1")
 
         case KafkaConfig.NumPartitionsProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "0")
         case KafkaConfig.LogDirsProp => // ignore string
diff --git a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
index e8d9c30a38364..1870a4996fea7 100644
--- a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala
@@ -170,8 +170,8 @@ object JaasTestUtils {
   }
 
   // Returns the dynamic configuration, using credentials for user #1
-  def clientLoginModule(mechanism: String, keytabLocation: Option[File]): String =
-    kafkaClientModule(mechanism, keytabLocation, KafkaClientPrincipal, KafkaPlainUser, KafkaPlainPassword, KafkaScramUser, KafkaScramPassword, KafkaOAuthBearerUser).toString
+  def clientLoginModule(mechanism: String, keytabLocation: Option[File], serviceName: String = serviceName): String =
+    kafkaClientModule(mechanism, keytabLocation, KafkaClientPrincipal, KafkaPlainUser, KafkaPlainPassword, KafkaScramUser, KafkaScramPassword, KafkaOAuthBearerUser, serviceName).toString
 
   def tokenClientLoginModule(tokenId: String, password: String): String = {
     ScramLoginModule(
@@ -223,10 +223,11 @@ object JaasTestUtils {
   }
 
   // consider refactoring if more mechanisms are added
-  private def kafkaClientModule(mechanism: String, 
+  private def kafkaClientModule(mechanism: String,
       keytabLocation: Option[File], clientPrincipal: String,
       plainUser: String, plainPassword: String, 
-      scramUser: String, scramPassword: String, oauthBearerUser: String): JaasModule = {
+      scramUser: String, scramPassword: String,
+      oauthBearerUser: String, serviceName: String = serviceName): JaasModule = {
     mechanism match {
       case "GSSAPI" =>
         Krb5LoginModule(

From f6890e78687afbbe09ff34b0a9383b548be77ee6 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 31 Aug 2018 13:10:27 -0700
Subject: [PATCH 0770/1847] MINOR: Next round of fetcher thread consolidation
 (#5587)

Pull the epoch request build logic up to `AbstractFetcherThread`. Also get rid of the `FetchRequest` indirection.

Reviewers: Ismael Juma , Rajini Sivaram 
---
 .../common/internals/PartitionStates.java     |   4 +
 .../kafka/server/AbstractFetcherThread.scala  | 158 ++++++++++++------
 .../server/ReplicaAlterLogDirsThread.scala    |  96 +++--------
 .../kafka/server/ReplicaFetcherThread.scala   | 118 ++++---------
 .../ReplicaFetcherThreadFatalErrorTest.scala  |   7 +-
 .../server/AbstractFetcherThreadTest.scala    |  53 +++---
 .../ReplicaAlterLogDirsThreadTest.scala       |  46 ++---
 7 files changed, 222 insertions(+), 260 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java b/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java
index 5b904c2671771..ba6563246a8c5 100644
--- a/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java
+++ b/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java
@@ -93,6 +93,10 @@ public List> partitionStates() {
         return result;
     }
 
+    public LinkedHashMap partitionStateMap() {
+        return new LinkedHashMap<>(map);
+    }
+
     /**
      * Returns the partition state values in order.
      */
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index fe9fc06885fda..e753f6e2fdc1b 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -39,7 +39,7 @@ import com.yammer.metrics.core.Gauge
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.{FatalExitError, PartitionStates}
 import org.apache.kafka.common.record.{FileRecords, MemoryRecords, Records}
-import org.apache.kafka.common.requests.{EpochEndOffset, FetchResponse}
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, FetchResponse}
 
 import scala.math._
 
@@ -54,7 +54,6 @@ abstract class AbstractFetcherThread(name: String,
                                      includeLogTruncation: Boolean)
   extends ShutdownableThread(name, isInterruptible) {
 
-  type REQ <: FetchRequest
   type PD = FetchResponse.PartitionData[Records]
 
   private[server] val partitionStates = new PartitionStates[PartitionFetchState]
@@ -74,18 +73,15 @@ abstract class AbstractFetcherThread(name: String,
   // handle a partition whose offset is out of range and return a new fetch offset
   protected def handleOffsetOutOfRange(topicPartition: TopicPartition): Long
 
-  // deal with partitions with errors, potentially due to leadership changes
-  protected def handlePartitionsWithErrors(partitions: Iterable[TopicPartition])
-
-  protected def buildLeaderEpochRequest(allPartitions: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[Map[TopicPartition, Int]]
-
   protected def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset]
 
-  protected def maybeTruncate(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]]
+  protected def truncate(topicPartition: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState
 
-  protected def buildFetchRequest(partitionMap: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[REQ]
+  protected def buildFetch(partitionMap: Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]]
 
-  protected def fetch(fetchRequest: REQ): Seq[(TopicPartition, PD)]
+  protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)]
+
+  protected def getReplica(tp: TopicPartition): Option[Replica]
 
   override def shutdown() {
     initiateShutdown()
@@ -99,34 +95,67 @@ abstract class AbstractFetcherThread(name: String,
     fetcherLagStats.unregister()
   }
 
-  private def states() = partitionStates.partitionStates.asScala.map { state => state.topicPartition -> state.value }
-
   override def doWork() {
     maybeTruncate()
-    val fetchRequest = inLock(partitionMapLock) {
-      val ResultWithPartitions(fetchRequest, partitionsWithError) = buildFetchRequest(states)
-      if (fetchRequest.isEmpty) {
+
+    val (fetchStates, fetchRequestOpt) = inLock(partitionMapLock) {
+      val fetchStates = partitionStates.partitionStateMap.asScala
+      val ResultWithPartitions(fetchRequestOpt, partitionsWithError) = buildFetch(fetchStates)
+
+      handlePartitionsWithErrors(partitionsWithError)
+
+      if (fetchRequestOpt.isEmpty) {
         trace(s"There are no active partitions. Back off for $fetchBackOffMs ms before sending a fetch request")
         partitionMapCond.await(fetchBackOffMs, TimeUnit.MILLISECONDS)
       }
-      handlePartitionsWithErrors(partitionsWithError)
-      fetchRequest
+
+      (fetchStates, fetchRequestOpt)
+    }
+
+    fetchRequestOpt.foreach { fetchRequest =>
+      processFetchRequest(fetchStates, fetchRequest)
+    }
+  }
+
+  // deal with partitions with errors, potentially due to leadership changes
+  private def handlePartitionsWithErrors(partitions: Iterable[TopicPartition]) {
+    if (partitions.nonEmpty)
+      delayPartitions(partitions, fetchBackOffMs)
+  }
+
+  /**
+   * Builds offset for leader epoch requests for partitions that are in the truncating phase based
+   * on latest epochs of the future replicas (the one that is fetching)
+   */
+  private def buildLeaderEpochRequest(): ResultWithPartitions[Map[TopicPartition, Int]] = inLock(partitionMapLock) {
+    var partitionsWithoutEpochs = mutable.Set.empty[TopicPartition]
+    var partitionsWithEpochs = mutable.Map.empty[TopicPartition, Int]
+
+    partitionStates.partitionStates.asScala.foreach { state =>
+      val tp = state.topicPartition
+      if (state.value.isTruncatingLog) {
+        getReplica(tp).flatMap(_.epochs).map(_.latestEpoch) match {
+          case Some(latestEpoch) => partitionsWithEpochs += tp -> latestEpoch
+          case None => partitionsWithoutEpochs += tp
+        }
+      }
     }
-    if (!fetchRequest.isEmpty)
-      processFetchRequest(fetchRequest)
+
+    debug(s"Build leaderEpoch request $partitionsWithEpochs")
+    ResultWithPartitions(partitionsWithEpochs, partitionsWithoutEpochs)
   }
 
   /**
     * - Build a leader epoch fetch based on partitions that are in the Truncating phase
-    * - Issue LeaderEpochRequeust, retrieving the latest offset for each partition's
+    * - Send OffsetsForLeaderEpochRequest, retrieving the latest offset for each partition's
     *   leader epoch. This is the offset the follower should truncate to ensure
     *   accurate log replication.
     * - Finally truncate the logs for partitions in the truncating phase and mark them
     *   truncation complete. Do this within a lock to ensure no leadership changes can
     *   occur during truncation.
     */
-  def maybeTruncate(): Unit = {
-    val ResultWithPartitions(epochRequests, partitionsWithError) = inLock(partitionMapLock) { buildLeaderEpochRequest(states) }
+  private def maybeTruncate(): Unit = {
+    val ResultWithPartitions(epochRequests, partitionsWithError) = buildLeaderEpochRequest()
     handlePartitionsWithErrors(partitionsWithError)
 
     if (epochRequests.nonEmpty) {
@@ -142,7 +171,31 @@ abstract class AbstractFetcherThread(name: String,
     }
   }
 
-  private def processFetchRequest(fetchRequest: REQ) {
+  private def maybeTruncate(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]] = {
+    val fetchOffsets = mutable.HashMap.empty[TopicPartition, OffsetTruncationState]
+    val partitionsWithError = mutable.Set[TopicPartition]()
+
+    fetchedEpochs.foreach { case (tp, leaderEpochOffset) =>
+      try {
+        if (leaderEpochOffset.hasError) {
+          info(s"Retrying leaderEpoch request for partition $tp as the leader reported an error: ${leaderEpochOffset.error}")
+          partitionsWithError += tp
+        } else {
+          val offsetTruncationState = truncate(tp, leaderEpochOffset)
+          fetchOffsets.put(tp, offsetTruncationState)
+        }
+      } catch {
+        case e: KafkaStorageException =>
+          info(s"Failed to truncate $tp", e)
+          partitionsWithError += tp
+      }
+    }
+
+    ResultWithPartitions(fetchOffsets, partitionsWithError)
+  }
+
+  private def processFetchRequest(fetchStates: Map[TopicPartition, PartitionFetchState],
+                                  fetchRequest: FetchRequest.Builder): Unit = {
     val partitionsWithError = mutable.Set[TopicPartition]()
     var responseData: Seq[(TopicPartition, PD)] = Seq.empty
 
@@ -169,13 +222,12 @@ abstract class AbstractFetcherThread(name: String,
       inLock(partitionMapLock) {
 
         responseData.foreach { case (topicPartition, partitionData) =>
-          val topic = topicPartition.topic
-          val partitionId = topicPartition.partition
-          Option(partitionStates.stateValue(topicPartition)).foreach(currentPartitionFetchState =>
+          Option(partitionStates.stateValue(topicPartition)).foreach { currentPartitionFetchState =>
             // It's possible that a partition is removed and re-added or truncated when there is a pending fetch request.
-            // In this case, we only want to process the fetch response if the partition state is ready for fetch and the current offset is the same as the offset requested.
-            if (fetchRequest.offset(topicPartition) == currentPartitionFetchState.fetchOffset &&
-                currentPartitionFetchState.isReadyForFetch) {
+            // In this case, we only want to process the fetch response if the partition state is ready for fetch and
+            // the current offset is the same as the offset requested.
+            val fetchOffset = fetchStates(topicPartition).fetchOffset
+            if (fetchOffset == currentPartitionFetchState.fetchOffset && currentPartitionFetchState.isReadyForFetch) {
               partitionData.error match {
                 case Errors.NONE =>
                   try {
@@ -183,7 +235,7 @@ abstract class AbstractFetcherThread(name: String,
                     val newOffset = records.batches.asScala.lastOption.map(_.nextOffset).getOrElse(
                       currentPartitionFetchState.fetchOffset)
 
-                    fetcherLagStats.getAndMaybePut(topic, partitionId).lag = Math.max(0L, partitionData.highWatermark - newOffset)
+                    fetcherLagStats.getAndMaybePut(topicPartition).lag = Math.max(0L, partitionData.highWatermark - newOffset)
                     // Once we hand off the partition data to the subclass, we can't mess with it any more in this thread
                     processPartitionData(topicPartition, currentPartitionFetchState.fetchOffset, partitionData, records)
 
@@ -232,7 +284,8 @@ abstract class AbstractFetcherThread(name: String,
                     partitionData.error.exception)
                   partitionsWithError += topicPartition
               }
-            })
+            }
+          }
         }
       }
     }
@@ -270,8 +323,11 @@ abstract class AbstractFetcherThread(name: String,
             new PartitionFetchState(initialFetchOffset, includeLogTruncation)
         tp -> fetchState
       }
-      val existingPartitionToState = states().toMap
-      partitionStates.set((existingPartitionToState ++ newPartitionToState).asJava)
+
+      newPartitionToState.foreach { case (tp, state) =>
+        partitionStates.updateAndMoveToEnd(tp, state)
+      }
+
       partitionMapCond.signalAll()
     } finally partitionMapLock.unlock()
   }
@@ -373,7 +429,8 @@ abstract class AbstractFetcherThread(name: String,
       for (partition <- partitions) {
         Option(partitionStates.stateValue(partition)).foreach (currentPartitionFetchState =>
           if (!currentPartitionFetchState.isDelayed)
-            partitionStates.updateAndMoveToEnd(partition, PartitionFetchState(currentPartitionFetchState.fetchOffset, new DelayedItem(delay), currentPartitionFetchState.truncatingLog))
+            partitionStates.updateAndMoveToEnd(partition, PartitionFetchState(currentPartitionFetchState.fetchOffset,
+              new DelayedItem(delay), currentPartitionFetchState.truncatingLog))
         )
       }
       partitionMapCond.signalAll()
@@ -385,7 +442,7 @@ abstract class AbstractFetcherThread(name: String,
     try {
       topicPartitions.foreach { topicPartition =>
         partitionStates.remove(topicPartition)
-        fetcherLagStats.unregister(topicPartition.topic, topicPartition.partition)
+        fetcherLagStats.unregister(topicPartition)
       }
     } finally partitionMapLock.unlock()
   }
@@ -397,8 +454,8 @@ abstract class AbstractFetcherThread(name: String,
   }
 
   private[server] def partitionsAndOffsets: Map[TopicPartition, BrokerAndInitialOffset] = inLock(partitionMapLock) {
-    partitionStates.partitionStates.asScala.map { case state =>
-      state.topicPartition -> new BrokerAndInitialOffset(sourceBroker, state.value.fetchOffset)
+    partitionStates.partitionStates.asScala.map { state =>
+      state.topicPartition -> BrokerAndInitialOffset(sourceBroker, state.value.fetchOffset)
     }.toMap
   }
 
@@ -418,11 +475,6 @@ object AbstractFetcherThread {
 
   case class ResultWithPartitions[R](result: R, partitionsWithError: Set[TopicPartition])
 
-  trait FetchRequest {
-    def isEmpty: Boolean
-    def offset(topicPartition: TopicPartition): Long
-  }
-
 }
 
 object FetcherMetrics {
@@ -436,8 +488,8 @@ class FetcherLagMetrics(metricId: ClientIdTopicPartition) extends KafkaMetricsGr
   private[this] val lagVal = new AtomicLong(-1L)
   private[this] val tags = Map(
     "clientId" -> metricId.clientId,
-    "topic" -> metricId.topic,
-    "partition" -> metricId.partitionId.toString)
+    "topic" -> metricId.topicPartition.topic,
+    "partition" -> metricId.topicPartition.partition.toString)
 
   newGauge(FetcherMetrics.ConsumerLag,
     new Gauge[Long] {
@@ -461,26 +513,26 @@ class FetcherLagStats(metricId: ClientIdAndBroker) {
   private val valueFactory = (k: ClientIdTopicPartition) => new FetcherLagMetrics(k)
   val stats = new Pool[ClientIdTopicPartition, FetcherLagMetrics](Some(valueFactory))
 
-  def getAndMaybePut(topic: String, partitionId: Int): FetcherLagMetrics = {
-    stats.getAndMaybePut(ClientIdTopicPartition(metricId.clientId, topic, partitionId))
+  def getAndMaybePut(topicPartition: TopicPartition): FetcherLagMetrics = {
+    stats.getAndMaybePut(ClientIdTopicPartition(metricId.clientId, topicPartition))
   }
 
-  def isReplicaInSync(topic: String, partitionId: Int): Boolean = {
-    val fetcherLagMetrics = stats.get(ClientIdTopicPartition(metricId.clientId, topic, partitionId))
+  def isReplicaInSync(topicPartition: TopicPartition): Boolean = {
+    val fetcherLagMetrics = stats.get(ClientIdTopicPartition(metricId.clientId, topicPartition))
     if (fetcherLagMetrics != null)
       fetcherLagMetrics.lag <= 0
     else
       false
   }
 
-  def unregister(topic: String, partitionId: Int) {
-    val lagMetrics = stats.remove(ClientIdTopicPartition(metricId.clientId, topic, partitionId))
+  def unregister(topicPartition: TopicPartition) {
+    val lagMetrics = stats.remove(ClientIdTopicPartition(metricId.clientId, topicPartition))
     if (lagMetrics != null) lagMetrics.unregister()
   }
 
   def unregister() {
     stats.keys.toBuffer.foreach { key: ClientIdTopicPartition =>
-      unregister(key.topic, key.partitionId)
+      unregister(key.topicPartition)
     }
   }
 }
@@ -501,8 +553,8 @@ class FetcherStats(metricId: ClientIdAndBroker) extends KafkaMetricsGroup {
 
 }
 
-case class ClientIdTopicPartition(clientId: String, topic: String, partitionId: Int) {
-  override def toString = "%s-%s-%d".format(clientId, topic, partitionId)
+case class ClientIdTopicPartition(clientId: String, topicPartition: TopicPartition) {
+  override def toString: String = s"$clientId-$topicPartition"
 }
 
 /**
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 05dc3566c0325..16212018b58a5 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -20,18 +20,16 @@ package kafka.server
 import java.util
 
 import kafka.api.Request
-import kafka.cluster.BrokerEndPoint
+import kafka.cluster.{BrokerEndPoint, Replica}
 import kafka.server.AbstractFetcherThread.ResultWithPartitions
 import kafka.server.QuotaFactory.UnboundedQuota
-import kafka.server.ReplicaAlterLogDirsThread.FetchRequest
-import kafka.server.epoch.LeaderEpochCache
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.KafkaStorageException
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.{MemoryRecords, Records}
 import org.apache.kafka.common.requests.EpochEndOffset._
 import org.apache.kafka.common.requests.FetchResponse.PartitionData
-import org.apache.kafka.common.requests.{EpochEndOffset, FetchResponse, FetchRequest => JFetchRequest}
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, FetchResponse}
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Seq, Set, mutable}
@@ -49,15 +47,17 @@ class ReplicaAlterLogDirsThread(name: String,
                                 isInterruptible = false,
                                 includeLogTruncation = true) {
 
-  type REQ = FetchRequest
-
   private val replicaId = brokerConfig.brokerId
   private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes
   private val fetchSize = brokerConfig.replicaFetchMaxBytes
 
-  def fetch(fetchRequest: FetchRequest): Seq[(TopicPartition, PD)] = {
+  protected def getReplica(tp: TopicPartition): Option[Replica] = {
+    replicaMgr.getReplica(tp, Request.FutureLocalReplicaId)
+  }
+
+  def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
     var partitionData: Seq[(TopicPartition, FetchResponse.PartitionData[Records])] = null
-    val request = fetchRequest.underlying.build()
+    val request = fetchRequest.build()
 
     def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]) {
       partitionData = responsePartitionData.map { case (tp, data) =>
@@ -129,28 +129,6 @@ class ReplicaAlterLogDirsThread(name: String,
     }
   }
 
-  def handlePartitionsWithErrors(partitions: Iterable[TopicPartition]) {
-    if (partitions.nonEmpty)
-      delayPartitions(partitions, brokerConfig.replicaFetchBackoffMs.toLong)
-  }
-
-  /**
-   * Builds offset for leader epoch requests for partitions that are in the truncating phase based
-   * on latest epochs of the future replicas (the one that is fetching)
-   */
-  def buildLeaderEpochRequest(allPartitions: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[Map[TopicPartition, Int]] = {
-    def epochCacheOpt(tp: TopicPartition): Option[LeaderEpochCache] = replicaMgr.getReplica(tp, Request.FutureLocalReplicaId).map(_.epochs.get)
-
-    val partitionEpochOpts = allPartitions
-      .filter { case (_, state) => state.isTruncatingLog }
-      .map { case (tp, _) => tp -> epochCacheOpt(tp) }.toMap
-
-    val (partitionsWithEpoch, partitionsWithoutEpoch) = partitionEpochOpts.partition { case (_, epochCacheOpt) => epochCacheOpt.nonEmpty }
-
-    val result = partitionsWithEpoch.map { case (tp, epochCacheOpt) => tp -> epochCacheOpt.get.latestEpoch }
-    ResultWithPartitions(result, partitionsWithoutEpoch.keys.toSet)
-  }
-
   /**
    * Fetches offset for leader epoch from local replica for each given topic partitions
    * @param partitions map of topic partition -> leader epoch of the future replica
@@ -183,34 +161,17 @@ class ReplicaAlterLogDirsThread(name: String,
    * the future replica may miss "mark for truncation" event and must use the offset for leader epoch
    * exchange with the current replica to truncate to the largest common log prefix for the topic partition
    */
-  def maybeTruncate(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]] = {
-    val fetchOffsets = scala.collection.mutable.HashMap.empty[TopicPartition, OffsetTruncationState]
-    val partitionsWithError = mutable.Set[TopicPartition]()
-
-    fetchedEpochs.foreach { case (topicPartition, epochOffset) =>
-      try {
-        val futureReplica = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId)
-        val partition = replicaMgr.getPartition(topicPartition).get
-
-        if (epochOffset.hasError) {
-          info(s"Retrying leaderEpoch request for partition $topicPartition as the current replica reported an error: ${epochOffset.error}")
-          partitionsWithError += topicPartition
-        } else {
-          val offsetTruncationState = getOffsetTruncationState(topicPartition, epochOffset, futureReplica, isFutureReplica = true)
+  override def truncate(topicPartition: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
+    val futureReplica = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId)
+    val partition = replicaMgr.getPartition(topicPartition).get
 
-          partition.truncateTo(offsetTruncationState.offset, isFuture = true)
-          fetchOffsets.put(topicPartition, offsetTruncationState)
-        }
-      } catch {
-        case e: KafkaStorageException =>
-          info(s"Failed to truncate $topicPartition", e)
-          partitionsWithError += topicPartition
-      }
-    }
-    ResultWithPartitions(fetchOffsets, partitionsWithError)
+    val offsetTruncationState = getOffsetTruncationState(topicPartition, epochEndOffset, futureReplica,
+      isFutureReplica = true)
+    partition.truncateTo(offsetTruncationState.offset, isFuture = true)
+    offsetTruncationState
   }
 
-  def buildFetchRequest(partitionMap: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[FetchRequest] = {
+  def buildFetch(partitionMap: Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
     // Only include replica in the fetch request if it is not throttled.
     val maxPartitionOpt = partitionMap.filter { case (_, partitionFetchState) =>
       partitionFetchState.isReadyForFetch && !quota.isQuotaExceeded
@@ -223,32 +184,29 @@ class ReplicaAlterLogDirsThread(name: String,
 
     // Only move one replica at a time to increase its catch-up rate and thus reduce the time spent on moving any given replica
     // Replicas are ordered by their TopicPartition
-    val requestMap = new util.LinkedHashMap[TopicPartition, JFetchRequest.PartitionData]
+    val requestMap = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
     val partitionsWithError = mutable.Set[TopicPartition]()
 
     if (maxPartitionOpt.nonEmpty) {
       val (topicPartition, partitionFetchState) = maxPartitionOpt.get
       try {
         val logStartOffset = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId).logStartOffset
-        requestMap.put(topicPartition, new JFetchRequest.PartitionData(partitionFetchState.fetchOffset, logStartOffset, fetchSize))
+        requestMap.put(topicPartition, new FetchRequest.PartitionData(partitionFetchState.fetchOffset, logStartOffset, fetchSize))
       } catch {
         case _: KafkaStorageException =>
           partitionsWithError += topicPartition
       }
     }
-    // Set maxWait and minBytes to 0 because the response should return immediately if
-    // the future log has caught up with the current log of the partition
-    val requestBuilder = JFetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion, replicaId, 0, 0, requestMap).setMaxBytes(maxBytes)
-    ResultWithPartitions(new FetchRequest(requestBuilder), partitionsWithError)
-  }
-}
 
-object ReplicaAlterLogDirsThread {
-
-  private[server] class FetchRequest(val underlying: JFetchRequest.Builder) extends AbstractFetcherThread.FetchRequest {
-    def isEmpty: Boolean = underlying.fetchData.isEmpty
-    def offset(topicPartition: TopicPartition): Long = underlying.fetchData.asScala(topicPartition).fetchOffset
-    override def toString = underlying.toString
+    val fetchRequestOpt = if (requestMap.isEmpty) {
+      None
+    } else {
+      // Set maxWait and minBytes to 0 because the response should return immediately if
+      // the future log has caught up with the current log of the partition
+      Some(FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion, replicaId, 0, 0, requestMap)
+        .setMaxBytes(maxBytes))
+    }
+    ResultWithPartitions(fetchRequestOpt, partitionsWithError)
   }
 
 }
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index 3b1a54f3bb184..5624e848cfc52 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -17,24 +17,20 @@
 
 package kafka.server
 
-import java.util
-
-import AbstractFetcherThread.ResultWithPartitions
 import kafka.api._
-import kafka.cluster.BrokerEndPoint
+import kafka.cluster.{BrokerEndPoint, Replica}
 import kafka.log.LogConfig
-import kafka.server.ReplicaFetcherThread._
-import kafka.server.epoch.LeaderEpochCache
+import kafka.server.AbstractFetcherThread.ResultWithPartitions
 import kafka.zk.AdminZkClient
 import org.apache.kafka.clients.FetchSessionHandler
-import org.apache.kafka.common.requests.EpochEndOffset._
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.KafkaStorageException
 import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record.{MemoryRecords, Records}
-import org.apache.kafka.common.requests.{EpochEndOffset, FetchResponse, ListOffsetRequest, ListOffsetResponse, OffsetsForLeaderEpochRequest, OffsetsForLeaderEpochResponse, FetchRequest => JFetchRequest}
+import org.apache.kafka.common.requests.EpochEndOffset._
+import org.apache.kafka.common.requests._
 import org.apache.kafka.common.utils.{LogContext, Time}
 
 import scala.collection.JavaConverters._
@@ -56,8 +52,6 @@ class ReplicaFetcherThread(name: String,
                                 isInterruptible = false,
                                 includeLogTruncation = true) {
 
-  type REQ = FetchRequest
-
   private val replicaId = brokerConfig.brokerId
   private val logContext = new LogContext(s"[ReplicaFetcher replicaId=$replicaId, leaderId=${sourceBroker.id}, " +
     s"fetcherId=$fetcherId] ")
@@ -90,7 +84,6 @@ class ReplicaFetcherThread(name: String,
     else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_1_IV2) 1
     else 0
 
-  private val fetchMetadataSupported = brokerConfig.interBrokerProtocolVersion >= KAFKA_1_1_IV0
   private val maxWait = brokerConfig.replicaFetchWaitMaxMs
   private val minBytes = brokerConfig.replicaFetchMinBytes
   private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes
@@ -99,7 +92,9 @@ class ReplicaFetcherThread(name: String,
 
   private val fetchSessionHandler = new FetchSessionHandler(logContext, sourceBroker.id)
 
-  private def epochCacheOpt(tp: TopicPartition): Option[LeaderEpochCache] =  replicaMgr.getReplica(tp).map(_.epochs.get)
+  protected def getReplica(tp: TopicPartition): Option[Replica] = {
+    replicaMgr.getReplica(tp)
+  }
 
   override def initiateShutdown(): Boolean = {
     val justShutdown = super.initiateShutdown()
@@ -228,15 +223,9 @@ class ReplicaFetcherThread(name: String,
     }
   }
 
-  // any logic for partitions whose leader has changed
-  def handlePartitionsWithErrors(partitions: Iterable[TopicPartition]) {
-    if (partitions.nonEmpty)
-      delayPartitions(partitions, brokerConfig.replicaFetchBackoffMs.toLong)
-  }
-
-  protected def fetch(fetchRequest: FetchRequest): Seq[(TopicPartition, PD)] = {
+  protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
     try {
-      val clientResponse = leaderEndpoint.sendRequest(fetchRequest.underlying)
+      val clientResponse = leaderEndpoint.sendRequest(fetchRequest)
       val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse[Records]]
       if (!fetchSessionHandler.handleResponse(fetchResponse)) {
         Nil
@@ -271,7 +260,7 @@ class ReplicaFetcherThread(name: String,
     }
   }
 
-  override def buildFetchRequest(partitionMap: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[FetchRequest] = {
+  override def buildFetch(partitionMap: Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
     val partitionsWithError = mutable.Set[TopicPartition]()
 
     val builder = fetchSessionHandler.newBuilder()
@@ -280,7 +269,7 @@ class ReplicaFetcherThread(name: String,
       if (partitionFetchState.isReadyForFetch && !shouldFollowerThrottle(quota, topicPartition)) {
         try {
           val logStartOffset = replicaMgr.getReplicaOrException(topicPartition).logStartOffset
-          builder.add(topicPartition, new JFetchRequest.PartitionData(
+          builder.add(topicPartition, new FetchRequest.PartitionData(
             partitionFetchState.fetchOffset, logStartOffset, fetchSize))
         } catch {
           case _: KafkaStorageException =>
@@ -292,63 +281,37 @@ class ReplicaFetcherThread(name: String,
     }
 
     val fetchData = builder.build()
-    val requestBuilder = JFetchRequest.Builder.
-      forReplica(fetchRequestVersion, replicaId, maxWait, minBytes, fetchData.toSend())
+    val fetchRequestOpt = if (fetchData.sessionPartitions.isEmpty && fetchData.toForget.isEmpty) {
+      None
+    } else {
+      val requestBuilder = FetchRequest.Builder
+        .forReplica(fetchRequestVersion, replicaId, maxWait, minBytes, fetchData.toSend)
         .setMaxBytes(maxBytes)
         .toForget(fetchData.toForget)
-    if (fetchMetadataSupported) {
-      requestBuilder.metadata(fetchData.metadata())
+        .metadata(fetchData.metadata)
+      Some(requestBuilder)
     }
-    ResultWithPartitions(new FetchRequest(fetchData.sessionPartitions(), requestBuilder), partitionsWithError)
+
+    ResultWithPartitions(fetchRequestOpt, partitionsWithError)
   }
 
   /**
    * Truncate the log for each partition's epoch based on leader's returned epoch and offset.
    * The logic for finding the truncation offset is implemented in AbstractFetcherThread.getOffsetTruncationState
    */
-  override def maybeTruncate(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]] = {
-    val fetchOffsets = scala.collection.mutable.HashMap.empty[TopicPartition, OffsetTruncationState]
-    val partitionsWithError = mutable.Set[TopicPartition]()
-
-    fetchedEpochs.foreach { case (tp, leaderEpochOffset) =>
-      try {
-        val replica = replicaMgr.getReplicaOrException(tp)
-        val partition = replicaMgr.getPartition(tp).get
-
-        if (leaderEpochOffset.hasError) {
-          info(s"Retrying leaderEpoch request for partition ${replica.topicPartition} as the leader reported an error: ${leaderEpochOffset.error}")
-          partitionsWithError += tp
-        } else {
-          val offsetTruncationState = getOffsetTruncationState(tp, leaderEpochOffset, replica)
-          if (offsetTruncationState.offset < replica.highWatermark.messageOffset)
-            warn(s"Truncating $tp to offset ${offsetTruncationState.offset} below high watermark ${replica.highWatermark.messageOffset}")
-
-          partition.truncateTo(offsetTruncationState.offset, isFuture = false)
-          // mark the future replica for truncation only when we do last truncation
-          if (offsetTruncationState.truncationCompleted)
-            replicaMgr.replicaAlterLogDirsManager.markPartitionsForTruncation(brokerConfig.brokerId, tp, offsetTruncationState.offset)
-          fetchOffsets.put(tp, offsetTruncationState)
-        }
-      } catch {
-        case e: KafkaStorageException =>
-          info(s"Failed to truncate $tp", e)
-          partitionsWithError += tp
-      }
-    }
-
-    ResultWithPartitions(fetchOffsets, partitionsWithError)
-  }
-
-  override def buildLeaderEpochRequest(allPartitions: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[Map[TopicPartition, Int]] = {
-    val partitionEpochOpts = allPartitions
-      .filter { case (_, state) => state.isTruncatingLog }
-      .map { case (tp, _) => tp -> epochCacheOpt(tp) }.toMap
-
-    val (partitionsWithEpoch, partitionsWithoutEpoch) = partitionEpochOpts.partition { case (_, epochCacheOpt) => epochCacheOpt.nonEmpty }
-
-    debug(s"Build leaderEpoch request $partitionsWithEpoch")
-    val result = partitionsWithEpoch.map { case (tp, epochCacheOpt) => tp -> epochCacheOpt.get.latestEpoch }
-    ResultWithPartitions(result, partitionsWithoutEpoch.keys.toSet)
+  override def truncate(tp: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
+    val replica = replicaMgr.getReplicaOrException(tp)
+    val partition = replicaMgr.getPartition(tp).get
+
+    val offsetTruncationState = getOffsetTruncationState(tp, epochEndOffset, replica)
+    if (offsetTruncationState.offset < replica.highWatermark.messageOffset)
+      warn(s"Truncating $tp to offset ${offsetTruncationState.offset} below high watermark ${replica.highWatermark.messageOffset}")
+    partition.truncateTo(offsetTruncationState.offset, isFuture = false)
+
+    // mark the future replica for truncation only when we do last truncation
+    if (offsetTruncationState.truncationCompleted)
+      replicaMgr.replicaAlterLogDirsManager.markPartitionsForTruncation(brokerConfig.brokerId, tp, offsetTruncationState.offset)
+    offsetTruncationState
   }
 
   override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
@@ -394,20 +357,7 @@ class ReplicaFetcherThread(name: String,
    *  the quota is exceeded and the replica is not in sync.
    */
   private def shouldFollowerThrottle(quota: ReplicaQuota, topicPartition: TopicPartition): Boolean = {
-    val isReplicaInSync = fetcherLagStats.isReplicaInSync(topicPartition.topic, topicPartition.partition)
+    val isReplicaInSync = fetcherLagStats.isReplicaInSync(topicPartition)
     quota.isThrottled(topicPartition) && quota.isQuotaExceeded && !isReplicaInSync
   }
 }
-
-object ReplicaFetcherThread {
-
-  private[server] class FetchRequest(val sessionParts: util.Map[TopicPartition, JFetchRequest.PartitionData],
-                                     val underlying: JFetchRequest.Builder)
-      extends AbstractFetcherThread.FetchRequest {
-    def offset(topicPartition: TopicPartition): Long =
-      sessionParts.get(topicPartition).fetchOffset
-    override def isEmpty = sessionParts.isEmpty && underlying.toForget().isEmpty
-    override def toString = underlying.toString
-  }
-
-}
diff --git a/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala b/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala
index 2f6db61f7b2f6..6fcf0ccd54b13 100644
--- a/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala
+++ b/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala
@@ -20,7 +20,6 @@ package kafka.server
 import java.util.concurrent.atomic.AtomicBoolean
 
 import kafka.cluster.BrokerEndPoint
-import kafka.server.ReplicaFetcherThread.FetchRequest
 import kafka.utils.{Exit, TestUtils}
 import kafka.utils.TestUtils.createBrokerConfigs
 import kafka.zk.ZooKeeperTestHarness
@@ -29,7 +28,7 @@ import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record.Records
-import org.apache.kafka.common.requests.FetchResponse
+import org.apache.kafka.common.requests.{FetchRequest, FetchResponse}
 import org.apache.kafka.common.utils.Time
 import org.junit.{After, Test}
 
@@ -89,8 +88,8 @@ class ReplicaFetcherThreadFatalErrorTest extends ZooKeeperTestHarness {
       import params._
       new ReplicaFetcherThread(threadName, fetcherId, sourceBroker, config, replicaManager, metrics, time, quotaManager) {
         override def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = throw new FatalExitError
-        override protected def fetch(fetchRequest: FetchRequest): Seq[(TopicPartition, PD)] = {
-          fetchRequest.underlying.fetchData.asScala.keys.toSeq.map { tp =>
+        override protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
+          fetchRequest.fetchData.asScala.keys.toSeq.map { tp =>
             (tp, new FetchResponse.PartitionData[Records](Errors.OFFSET_OUT_OF_RANGE,
               FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET,
               FetchResponse.INVALID_LOG_START_OFFSET, null, null))
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
index db98a87f03218..15abc6896b1d9 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
@@ -19,13 +19,12 @@ package kafka.server
 
 import AbstractFetcherThread._
 import com.yammer.metrics.Metrics
-import kafka.cluster.BrokerEndPoint
-import kafka.server.AbstractFetcherThread.FetchRequest
+import kafka.cluster.{BrokerEndPoint, Replica}
 import kafka.utils.TestUtils
 import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.record.{CompressionType, MemoryRecords, Records, SimpleRecord}
-import org.apache.kafka.common.requests.EpochEndOffset
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest}
 import org.apache.kafka.common.requests.FetchResponse.PartitionData
 import org.junit.Assert.{assertFalse, assertTrue}
 import org.junit.{Before, Test}
@@ -87,19 +86,23 @@ class AbstractFetcherThreadTest {
 
   private def allMetricsNames = Metrics.defaultRegistry().allMetrics().asScala.keySet.map(_.getName)
 
-  class DummyFetchRequest(val offsets: collection.Map[TopicPartition, Long]) extends FetchRequest {
-    override def isEmpty: Boolean = offsets.isEmpty
-
-    override def offset(topicPartition: TopicPartition): Long = offsets(topicPartition)
+  protected def fetchRequestBuilder(partitionMap: collection.Map[TopicPartition, PartitionFetchState]): FetchRequest.Builder = {
+    val partitionData = partitionMap.map { case (tp, fetchState) =>
+      tp -> new FetchRequest.PartitionData(fetchState.fetchOffset, 0, 1024 * 1024)
+    }.toMap.asJava
+    FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion, 0, 0, 1, partitionData)
   }
 
   class DummyFetcherThread(name: String,
                            clientId: String,
                            sourceBroker: BrokerEndPoint,
                            fetchBackOffMs: Int = 0)
-    extends AbstractFetcherThread(name, clientId, sourceBroker, fetchBackOffMs, isInterruptible = true, includeLogTruncation = false) {
+    extends AbstractFetcherThread(name, clientId, sourceBroker,
+      fetchBackOffMs,
+      isInterruptible = true,
+      includeLogTruncation = false) {
 
-    type REQ = DummyFetchRequest
+    protected def getReplica(tp: TopicPartition): Option[Replica] = None
 
     override def processPartitionData(topicPartition: TopicPartition,
                                       fetchOffset: Long,
@@ -108,27 +111,21 @@ class AbstractFetcherThreadTest {
 
     override def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = 0L
 
-    override def handlePartitionsWithErrors(partitions: Iterable[TopicPartition]): Unit = {}
-
-    override protected def fetch(fetchRequest: DummyFetchRequest): Seq[(TopicPartition, PD)] =
-      fetchRequest.offsets.mapValues(_ => new PartitionData[Records](Errors.NONE, 0, 0, 0,
+    override protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] =
+      fetchRequest.fetchData.asScala.mapValues(_ => new PartitionData[Records](Errors.NONE, 0, 0, 0,
         Seq.empty.asJava, MemoryRecords.EMPTY)).toSeq
 
-    override protected def buildFetchRequest(partitionMap: collection.Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[DummyFetchRequest] =
-      ResultWithPartitions(new DummyFetchRequest(partitionMap.map { case (k, v) => (k, v.fetchOffset) }.toMap), Set())
-
-    override def buildLeaderEpochRequest(allPartitions: Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[Map[TopicPartition, Int]] = {
-      ResultWithPartitions(Map(), Set())
+    override protected def buildFetch(partitionMap: collection.Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
+      ResultWithPartitions(Some(fetchRequestBuilder(partitionMap)), Set())
     }
 
     override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = { Map() }
 
-    override def maybeTruncate(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, OffsetTruncationState]] = {
-      ResultWithPartitions(Map(), Set())
+    override def truncate(tp: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
+      OffsetTruncationState(epochEndOffset.endOffset, truncationCompleted = true)
     }
   }
 
-
   @Test
   def testFetchRequestCorruptedMessageException() {
     val partition = new TopicPartition("topic", 0)
@@ -182,7 +179,7 @@ class AbstractFetcherThreadTest {
       }
     }
 
-    override protected def fetch(fetchRequest: DummyFetchRequest): Seq[(TopicPartition, PD)] = {
+    override protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
       fetchCount += 1
       // Set the first fetch to get a corrupted message
       if (fetchCount == 1) {
@@ -193,26 +190,24 @@ class AbstractFetcherThreadTest {
         // flip some bits in the message to ensure the crc fails
         buffer.putInt(15, buffer.getInt(15) ^ 23422)
         buffer.putInt(30, buffer.getInt(30) ^ 93242)
-        fetchRequest.offsets.mapValues(_ => new PartitionData[Records](Errors.NONE, 0L, 0L, 0L,
+        fetchRequest.fetchData.asScala.mapValues(_ => new PartitionData[Records](Errors.NONE, 0L, 0L, 0L,
           Seq.empty.asJava, records)).toSeq
       } else {
         // Then, the following fetches get the normal data
-        fetchRequest.offsets.mapValues(v => normalPartitionDataSet(v.toInt)).toSeq
+        fetchRequest.fetchData.asScala.mapValues(v => normalPartitionDataSet(v.fetchOffset.toInt)).toSeq
       }
     }
 
-    override protected def buildFetchRequest(partitionMap: collection.Seq[(TopicPartition, PartitionFetchState)]): ResultWithPartitions[DummyFetchRequest] = {
+    override protected def buildFetch(partitionMap: collection.Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
       val requestMap = new mutable.HashMap[TopicPartition, Long]
       partitionMap.foreach { case (topicPartition, partitionFetchState) =>
         // Add backoff delay check
         if (partitionFetchState.isReadyForFetch)
           requestMap.put(topicPartition, partitionFetchState.fetchOffset)
       }
-      ResultWithPartitions(new DummyFetchRequest(requestMap), Set())
+      ResultWithPartitions(Some(fetchRequestBuilder(partitionMap)), Set())
     }
 
-    override def handlePartitionsWithErrors(partitions: Iterable[TopicPartition]) = delayPartitions(partitions, fetchBackOffMs.toLong)
-
   }
 
 }
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
index 29a1c9f069729..8fb5ab6feba98 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala
@@ -479,12 +479,15 @@ class ReplicaAlterLogDirsThreadTest {
       brokerTopicStats = null)
     thread.addPartitions(Map(t1p0 -> 0, t1p1 -> 0))
 
-    val ResultWithPartitions(fetchRequest, partitionsWithError) =
-      thread.buildFetchRequest(Seq((t1p0, new PartitionFetchState(150)), (t1p1, new PartitionFetchState(160))))
+    val ResultWithPartitions(fetchRequestOpt, partitionsWithError) = thread.buildFetch(Map(
+      t1p0 -> new PartitionFetchState(150),
+      t1p1 -> new PartitionFetchState(160)))
 
-    assertFalse(fetchRequest.isEmpty)
+    assertTrue(fetchRequestOpt.isDefined)
+    val fetchRequest = fetchRequestOpt.get
+    assertFalse(fetchRequest.fetchData.isEmpty)
     assertFalse(partitionsWithError.nonEmpty)
-    val request = fetchRequest.underlying.build()
+    val request = fetchRequest.build()
     assertEquals(0, request.minBytes)
     val fetchInfos = request.fetchData.asScala.toSeq
     assertEquals(1, fetchInfos.length)
@@ -523,37 +526,38 @@ class ReplicaAlterLogDirsThreadTest {
     thread.addPartitions(Map(t1p0 -> 0, t1p1 -> 0))
 
     // one partition is ready and one is truncating
-    val ResultWithPartitions(fetchRequest, partitionsWithError) =
-      thread.buildFetchRequest(Seq(
-        (t1p0, new PartitionFetchState(150)),
-        (t1p1, new PartitionFetchState(160, truncatingLog=true))))
+    val ResultWithPartitions(fetchRequestOpt, partitionsWithError) = thread.buildFetch(Map(
+        t1p0 -> new PartitionFetchState(150),
+        t1p1 -> new PartitionFetchState(160, truncatingLog=true)))
 
-    assertFalse(fetchRequest.isEmpty)
+    assertTrue(fetchRequestOpt.isDefined)
+    val fetchRequest = fetchRequestOpt.get
+    assertFalse(fetchRequest.fetchData.isEmpty)
     assertFalse(partitionsWithError.nonEmpty)
-    val fetchInfos = fetchRequest.underlying.build().fetchData.asScala.toSeq
+    val fetchInfos = fetchRequest.build().fetchData.asScala.toSeq
     assertEquals(1, fetchInfos.length)
     assertEquals("Expected fetch request for non-truncating partition", t1p0, fetchInfos.head._1)
     assertEquals(150, fetchInfos.head._2.fetchOffset)
 
     // one partition is ready and one is delayed
-    val ResultWithPartitions(fetchRequest2, partitionsWithError2) =
-      thread.buildFetchRequest(Seq(
-        (t1p0, new PartitionFetchState(140)),
-        (t1p1, new PartitionFetchState(160, delay=new DelayedItem(5000)))))
+    val ResultWithPartitions(fetchRequest2Opt, partitionsWithError2) = thread.buildFetch(Map(
+        t1p0 -> new PartitionFetchState(140),
+        t1p1 -> new PartitionFetchState(160, delay=new DelayedItem(5000))))
 
-    assertFalse(fetchRequest2.isEmpty)
+    assertTrue(fetchRequest2Opt.isDefined)
+    val fetchRequest2 = fetchRequest2Opt.get
+    assertFalse(fetchRequest2.fetchData.isEmpty)
     assertFalse(partitionsWithError2.nonEmpty)
-    val fetchInfos2 = fetchRequest2.underlying.build().fetchData.asScala.toSeq
+    val fetchInfos2 = fetchRequest2.build().fetchData.asScala.toSeq
     assertEquals(1, fetchInfos2.length)
     assertEquals("Expected fetch request for non-delayed partition", t1p0, fetchInfos2.head._1)
     assertEquals(140, fetchInfos2.head._2.fetchOffset)
 
     // both partitions are delayed
-    val ResultWithPartitions(fetchRequest3, partitionsWithError3) =
-      thread.buildFetchRequest(Seq(
-        (t1p0, new PartitionFetchState(140, delay=new DelayedItem(5000))),
-        (t1p1, new PartitionFetchState(160, delay=new DelayedItem(5000)))))
-    assertTrue("Expected no fetch requests since all partitions are delayed", fetchRequest3.isEmpty)
+    val ResultWithPartitions(fetchRequest3Opt, partitionsWithError3) = thread.buildFetch(Map(
+        t1p0 -> new PartitionFetchState(140, delay=new DelayedItem(5000)),
+        t1p1 -> new PartitionFetchState(160, delay=new DelayedItem(5000))))
+    assertTrue("Expected no fetch requests since all partitions are delayed", fetchRequest3Opt.isEmpty)
     assertFalse(partitionsWithError3.nonEmpty)
   }
 

From d57fe1b053546966e6a867d84ee24dd256bb071a Mon Sep 17 00:00:00 2001
From: John Roesler 
Date: Fri, 31 Aug 2018 15:13:42 -0500
Subject: [PATCH 0771/1847] MINOR: single Jackson serde for PageViewTypedDemo
 (#5590)

Previously, we depicted creating a Jackson serde for every pojo class, which becomes a burden in practice. There are many ways to avoid this and just have a single serde, so we've decided to model this design choice instead.

Reviewers: Viktor Somogyi , Bill Bejeck , Guozhang Wang 
---
 checkstyle/import-control.xml                 |   2 +-
 gradle/findbugs-exclude.xml                   |  17 ++
 .../pageview/JsonPOJODeserializer.java        |  61 ------
 .../examples/pageview/JsonPOJOSerializer.java |  55 ------
 .../examples/pageview/PageViewTypedDemo.java  | 175 +++++++++++-------
 5 files changed, 128 insertions(+), 182 deletions(-)
 delete mode 100644 streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJODeserializer.java
 delete mode 100644 streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJOSerializer.java

diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 840e5516e072a..bd5c11fc579bc 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -216,7 +216,7 @@
     
 
     
-      
+      
       
     
 
diff --git a/gradle/findbugs-exclude.xml b/gradle/findbugs-exclude.xml
index 09981852f951c..62240d84c730f 100644
--- a/gradle/findbugs-exclude.xml
+++ b/gradle/findbugs-exclude.xml
@@ -292,4 +292,21 @@ For a detailed description of findbugs bug categories, see http://findbugs.sourc
         
         
     
+
+    
+    
+        
+        
+    
+    
+        
+        
+    
+    
+        
+        
+    
+
+    
+
 
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJODeserializer.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJODeserializer.java
deleted file mode 100644
index d55246c9bd9b3..0000000000000
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJODeserializer.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.examples.pageview;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.kafka.common.errors.SerializationException;
-import org.apache.kafka.common.serialization.Deserializer;
-
-import java.util.Map;
-
-public class JsonPOJODeserializer implements Deserializer {
-    private ObjectMapper objectMapper = new ObjectMapper();
-
-    private Class tClass;
-
-    /**
-     * Default constructor needed by Kafka
-     */
-    public JsonPOJODeserializer() {
-    }
-
-    @SuppressWarnings("unchecked")
-    @Override
-    public void configure(final Map props, final boolean isKey) {
-        tClass = (Class) props.get("JsonPOJOClass");
-    }
-
-    @Override
-    public T deserialize(final String topic, final byte[] bytes) {
-        if (bytes == null)
-            return null;
-
-        final T data;
-        try {
-            data = objectMapper.readValue(bytes, tClass);
-        } catch (final Exception e) {
-            throw new SerializationException(e);
-        }
-
-        return data;
-    }
-
-    @Override
-    public void close() {
-
-    }
-}
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJOSerializer.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJOSerializer.java
deleted file mode 100644
index 81ccf1ed066e0..0000000000000
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/JsonPOJOSerializer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.examples.pageview;
-
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.kafka.common.errors.SerializationException;
-import org.apache.kafka.common.serialization.Serializer;
-
-import java.util.Map;
-
-public class JsonPOJOSerializer implements Serializer {
-    private final ObjectMapper objectMapper = new ObjectMapper();
-
-    /**
-     * Default constructor needed by Kafka
-     */
-    public JsonPOJOSerializer() {
-    }
-    
-    @Override
-    public void configure(final Map props, final boolean isKey) {
-    }
-
-    @Override
-    public byte[] serialize(final String topic, final T data) {
-        if (data == null)
-            return null;
-
-        try {
-            return objectMapper.writeValueAsBytes(data);
-        } catch (final Exception e) {
-            throw new SerializationException("Error serializing JSON message", e);
-        }
-    }
-
-    @Override
-    public void close() {
-    }
-
-}
diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java
index 503dbebae65a6..871d83606f17b 100644
--- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java
+++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/pageview/PageViewTypedDemo.java
@@ -16,23 +16,26 @@
  */
 package org.apache.kafka.streams.examples.pageview;
 
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.KTable;
-import org.apache.kafka.streams.kstream.Produced;
 import org.apache.kafka.streams.kstream.Serialized;
 import org.apache.kafka.streams.kstream.TimeWindows;
 
-import java.util.HashMap;
+import java.io.IOException;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.CountDownLatch;
@@ -48,35 +51,122 @@
  * is JSON string representing a record in the stream or table, to compute the number of pageviews per user region.
  *
  * Before running this example you must create the input topics and the output topic (e.g. via
- * bin/kafka-topics.sh --create ...), and write some data to the input topics (e.g. via
- * bin/kafka-console-producer.sh). Otherwise you won't see any data arriving in the output topic.
+ * bin/kafka-topics --create ...), and write some data to the input topics (e.g. via
+ * bin/kafka-console-producer). Otherwise you won't see any data arriving in the output topic.
+ *
+ * The inputs for this example are:
+ * - Topic: streams-pageview-input
+ *   Key Format: (String) USER_ID
+ *   Value Format: (JSON) {"_t": "pv", "user": (String USER_ID), "page": (String PAGE_ID), "timestamp": (long ms TIMESTAMP)}
+ *
+ * - Topic: streams-userprofile-input
+ *   Key Format: (String) USER_ID
+ *   Value Format: (JSON) {"_t": "up", "region": (String REGION), "timestamp": (long ms TIMESTAMP)}
+ *
+ * To observe the results, read the output topic (e.g., via bin/kafka-console-consumer)
+ * - Topic: streams-pageviewstats-typed-output
+ *   Key Format: (JSON) {"_t": "wpvbr", "windowStart": (long ms WINDOW_TIMESTAMP), "region": (String REGION)}
+ *   Value Format: (JSON) {"_t": "rc", "count": (long REGION_COUNT), "region": (String REGION)}
+ *
+ * Note, the "_t" field is necessary to help Jackson identify the correct class for deserialization in the
+ * generic {@link JSONSerde}. If you instead specify a specific serde per class, you won't need the extra "_t" field.
  */
+@SuppressWarnings({"WeakerAccess", "unused"})
 public class PageViewTypedDemo {
 
+    /**
+     * A serde for any class that implements {@link JSONSerdeCompatible}. Note that the classes also need to
+     * be registered in the {@code @JsonSubTypes} annotation on {@link JSONSerdeCompatible}.
+     *
+     * @param  The concrete type of the class that gets de/serialized
+     */
+    public static class JSONSerde implements Serializer, Deserializer, Serde {
+        private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+        @Override
+        public void configure(final Map configs, final boolean isKey) {}
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public T deserialize(final String topic, final byte[] data) {
+            if (data == null) {
+                return null;
+            }
+
+            try {
+                return (T) OBJECT_MAPPER.readValue(data, JSONSerdeCompatible.class);
+            } catch (final IOException e) {
+                throw new SerializationException(e);
+            }
+        }
+
+        @Override
+        public byte[] serialize(final String topic, final T data) {
+            if (data == null) {
+                return null;
+            }
+
+            try {
+                return OBJECT_MAPPER.writeValueAsBytes(data);
+            } catch (final Exception e) {
+                throw new SerializationException("Error serializing JSON message", e);
+            }
+        }
+
+        @Override
+        public void close() {}
+
+        @Override
+        public Serializer serializer() {
+            return this;
+        }
+
+        @Override
+        public Deserializer deserializer() {
+            return this;
+        }
+    }
+
+    /**
+     * An interface for registering types that can be de/serialized with {@link JSONSerde}.
+     */
+    @SuppressWarnings("DefaultAnnotationParam") // being explicit for the example
+    @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "_t")
+    @JsonSubTypes({
+                      @JsonSubTypes.Type(value = PageView.class, name = "pv"),
+                      @JsonSubTypes.Type(value = UserProfile.class, name = "up"),
+                      @JsonSubTypes.Type(value = PageViewByRegion.class, name = "pvbr"),
+                      @JsonSubTypes.Type(value = WindowedPageViewByRegion.class, name = "wpvbr"),
+                      @JsonSubTypes.Type(value = RegionCount.class, name = "rc")
+                  })
+    public interface JSONSerdeCompatible {
+
+    }
+
     // POJO classes
-    static public class PageView {
+    static public class PageView implements JSONSerdeCompatible {
         public String user;
         public String page;
         public Long timestamp;
     }
 
-    static public class UserProfile {
+    static public class UserProfile implements JSONSerdeCompatible {
         public String region;
         public Long timestamp;
     }
 
-    static public class PageViewByRegion {
+    static public class PageViewByRegion implements JSONSerdeCompatible {
         public String user;
         public String page;
         public String region;
     }
 
-    static public class WindowedPageViewByRegion {
+    static public class WindowedPageViewByRegion implements JSONSerdeCompatible {
         public long windowStart;
         public String region;
     }
 
-    static public class RegionCount {
+    static public class RegionCount implements JSONSerdeCompatible {
         public long count;
         public String region;
     }
@@ -86,67 +176,21 @@ public static void main(final String[] args) {
         props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
         props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
         props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
+        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, JSONSerde.class);
+        props.put(StreamsConfig.DEFAULT_WINDOWED_KEY_SERDE_INNER_CLASS, JSONSerde.class);
+        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, JSONSerde.class);
+        props.put(StreamsConfig.DEFAULT_WINDOWED_VALUE_SERDE_INNER_CLASS, JSONSerde.class);
         props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
+        props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
 
         // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
         props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
 
         final StreamsBuilder builder = new StreamsBuilder();
 
-        // TODO: the following can be removed with a serialization factory
-        final Map serdeProps = new HashMap<>();
-
-        final Serializer pageViewSerializer = new JsonPOJOSerializer<>();
-        serdeProps.put("JsonPOJOClass", PageView.class);
-        pageViewSerializer.configure(serdeProps, false);
-
-        final Deserializer pageViewDeserializer = new JsonPOJODeserializer<>();
-        serdeProps.put("JsonPOJOClass", PageView.class);
-        pageViewDeserializer.configure(serdeProps, false);
-
-        final Serde pageViewSerde = Serdes.serdeFrom(pageViewSerializer, pageViewDeserializer);
-
-        final Serializer userProfileSerializer = new JsonPOJOSerializer<>();
-        serdeProps.put("JsonPOJOClass", UserProfile.class);
-        userProfileSerializer.configure(serdeProps, false);
-
-        final Deserializer userProfileDeserializer = new JsonPOJODeserializer<>();
-        serdeProps.put("JsonPOJOClass", UserProfile.class);
-        userProfileDeserializer.configure(serdeProps, false);
-
-        final Serde userProfileSerde = Serdes.serdeFrom(userProfileSerializer, userProfileDeserializer);
-
-        final Serializer wPageViewByRegionSerializer = new JsonPOJOSerializer<>();
-        serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
-        wPageViewByRegionSerializer.configure(serdeProps, false);
-
-        final Deserializer wPageViewByRegionDeserializer = new JsonPOJODeserializer<>();
-        serdeProps.put("JsonPOJOClass", WindowedPageViewByRegion.class);
-        wPageViewByRegionDeserializer.configure(serdeProps, false);
-
-        final Serde wPageViewByRegionSerde = Serdes.serdeFrom(wPageViewByRegionSerializer, wPageViewByRegionDeserializer);
-
-        final Serializer regionCountSerializer = new JsonPOJOSerializer<>();
-        serdeProps.put("JsonPOJOClass", RegionCount.class);
-        regionCountSerializer.configure(serdeProps, false);
-
-        final Deserializer regionCountDeserializer = new JsonPOJODeserializer<>();
-        serdeProps.put("JsonPOJOClass", RegionCount.class);
-        regionCountDeserializer.configure(serdeProps, false);
-        final Serde regionCountSerde = Serdes.serdeFrom(regionCountSerializer, regionCountDeserializer);
-
-        final Serializer pageViewByRegionSerializer = new JsonPOJOSerializer<>();
-        serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
-        pageViewByRegionSerializer.configure(serdeProps, false);
-        final Deserializer pageViewByRegionDeserializer = new JsonPOJODeserializer<>();
-        serdeProps.put("JsonPOJOClass", PageViewByRegion.class);
-        pageViewByRegionDeserializer.configure(serdeProps, false);
-        final Serde pageViewByRegionSerde = Serdes.serdeFrom(pageViewByRegionSerializer, pageViewByRegionDeserializer);
-
-        final KStream views = builder.stream("streams-pageview-input", Consumed.with(Serdes.String(), pageViewSerde));
+        final KStream views = builder.stream("streams-pageview-input", Consumed.with(Serdes.String(), new JSONSerde<>()));
 
-        final KTable users = builder.table("streams-userprofile-input",
-                                                                Consumed.with(Serdes.String(), userProfileSerde));
+        final KTable users = builder.table("streams-userprofile-input", Consumed.with(Serdes.String(), new JSONSerde<>()));
 
         final KStream regionCount = views
             .leftJoin(users, (view, profile) -> {
@@ -162,7 +206,7 @@ public static void main(final String[] args) {
                 return viewByRegion;
             })
             .map((user, viewRegion) -> new KeyValue<>(viewRegion.region, viewRegion))
-            .groupByKey(Serialized.with(Serdes.String(), pageViewByRegionSerde))
+            .groupByKey(Serialized.with(Serdes.String(), new JSONSerde<>()))
             .windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(7)).advanceBy(TimeUnit.SECONDS.toMillis(1)))
             .count()
             .toStream()
@@ -179,7 +223,7 @@ public static void main(final String[] args) {
             });
 
         // write to the result topic
-        regionCount.to("streams-pageviewstats-typed-output", Produced.with(wPageViewByRegionSerde, regionCountSerde));
+        regionCount.to("streams-pageviewstats-typed-output");
 
         final KafkaStreams streams = new KafkaStreams(builder.build(), props);
         final CountDownLatch latch = new CountDownLatch(1);
@@ -197,6 +241,7 @@ public void run() {
             streams.start();
             latch.await();
         } catch (final Throwable e) {
+            e.printStackTrace();
             System.exit(1);
         }
         System.exit(0);

From 4f38c8cd6079100548ca8056c358a2fea57986c2 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Fri, 31 Aug 2018 18:40:42 -0700
Subject: [PATCH 0772/1847] KAFKA-7369; Handle retriable errors in AdminClient
 list groups API (#5595)

We should retry when possible if ListGroups fails due to a retriable error (e.g. coordinator loading).

Reviewers: Colin Patrick McCabe ,  Guozhang Wang 
---
 .../kafka/clients/admin/KafkaAdminClient.java |  7 +++-
 .../common/requests/ListGroupsResponse.java   |  1 +
 .../clients/admin/KafkaAdminClientTest.java   | 40 ++++++++++++++++---
 3 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 8ddb0c08627ac..904cd0601e5b1 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -2567,8 +2567,11 @@ private void maybeAddConsumerGroup(ListGroupsResponse.Group group) {
                         void handleResponse(AbstractResponse abstractResponse) {
                             final ListGroupsResponse response = (ListGroupsResponse) abstractResponse;
                             synchronized (results) {
-                                if (response.error() != Errors.NONE) {
-                                    results.addError(response.error().exception(), node);
+                                Errors error = response.error();
+                                if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.COORDINATOR_NOT_AVAILABLE) {
+                                    throw error.exception();
+                                } else if (error != Errors.NONE) {
+                                    results.addError(error.exception(), node);
                                 } else {
                                     for (ListGroupsResponse.Group group : response.groups()) {
                                         maybeAddConsumerGroup(group);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
index b108803590f17..af6f7212e8cd5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
@@ -62,6 +62,7 @@ public static Schema[] schemaVersions() {
     /**
      * Possible error codes:
      *
+     * COORDINATOR_LOADING_IN_PROGRESS (14)
      * COORDINATOR_NOT_AVAILABLE (15)
      * AUTHORIZATION_FAILED (29)
      */
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 0245cbd36950a..c0dc542b15954 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -36,7 +36,6 @@
 import org.apache.kafka.common.acl.AclPermissionType;
 import org.apache.kafka.common.config.ConfigResource;
 import org.apache.kafka.common.errors.AuthenticationException;
-import org.apache.kafka.common.errors.CoordinatorNotAvailableException;
 import org.apache.kafka.common.errors.GroupAuthorizationException;
 import org.apache.kafka.common.errors.InvalidTopicException;
 import org.apache.kafka.common.errors.LeaderNotAvailableException;
@@ -46,6 +45,7 @@
 import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.TopicDeletionDisabledException;
+import org.apache.kafka.common.errors.UnknownServerException;
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.ApiError;
@@ -863,9 +863,11 @@ public void testListConsumerGroups() throws Exception {
         Node node0 = new Node(0, "localhost", 8121);
         Node node1 = new Node(1, "localhost", 8122);
         Node node2 = new Node(2, "localhost", 8123);
+        Node node3 = new Node(3, "localhost", 8124);
         nodes.put(0, node0);
         nodes.put(1, node1);
         nodes.put(2, node2);
+        nodes.put(3, node3);
 
         final Cluster cluster = new Cluster(
                 "mockClusterId",
@@ -902,13 +904,19 @@ public void testListConsumerGroups() throws Exception {
                             )),
                     node0);
 
+            // handle retriable errors
             env.kafkaClient().prepareResponseFrom(
                     new ListGroupsResponse(
                             Errors.COORDINATOR_NOT_AVAILABLE,
                             Collections.emptyList()
                     ),
                     node1);
-
+            env.kafkaClient().prepareResponseFrom(
+                    new ListGroupsResponse(
+                            Errors.COORDINATOR_LOAD_IN_PROGRESS,
+                            Collections.emptyList()
+                    ),
+                    node1);
             env.kafkaClient().prepareResponseFrom(
                     new ListGroupsResponse(
                             Errors.NONE,
@@ -916,15 +924,37 @@ public void testListConsumerGroups() throws Exception {
                                     new ListGroupsResponse.Group("group-2", ConsumerProtocol.PROTOCOL_TYPE),
                                     new ListGroupsResponse.Group("group-connect-2", "connector")
                             )),
+                    node1);
+
+            env.kafkaClient().prepareResponseFrom(
+                    new ListGroupsResponse(
+                            Errors.NONE,
+                            asList(
+                                    new ListGroupsResponse.Group("group-3", ConsumerProtocol.PROTOCOL_TYPE),
+                                    new ListGroupsResponse.Group("group-connect-3", "connector")
+                            )),
                     node2);
 
+            // fatal error
+            env.kafkaClient().prepareResponseFrom(
+                    new ListGroupsResponse(
+                            Errors.UNKNOWN_SERVER_ERROR,
+                            Collections.emptyList()),
+                    node3);
+
+
             final ListConsumerGroupsResult result = env.adminClient().listConsumerGroups();
-            TestUtils.assertFutureError(result.all(), CoordinatorNotAvailableException.class);
+            TestUtils.assertFutureError(result.all(), UnknownServerException.class);
+
             Collection listings = result.valid().get();
-            assertEquals(2, listings.size());
+            assertEquals(3, listings.size());
+
+            Set groupIds = new HashSet<>();
             for (ConsumerGroupListing listing : listings) {
-                assertTrue(listing.groupId().equals("group-1") || listing.groupId().equals("group-2"));
+                groupIds.add(listing.groupId());
             }
+
+            assertEquals(Utils.mkSet("group-1", "group-2", "group-3"), groupIds);
             assertEquals(1, result.errors().get().size());
         }
     }

From aa7358e8cc0305dd9051fe0d281913247a39264d Mon Sep 17 00:00:00 2001
From: huxi 
Date: Sun, 2 Sep 2018 00:48:50 +0800
Subject: [PATCH 0773/1847] KAFKA-7326: KStream.print() should flush on each
 line for PrintStream (#5579)

Reviewers: Matthias J. Sax , Guozhang Wang , Bill Bejeck , Kamal Chandraprakash 
---
 docs/streams/developer-guide/dsl-api.html                      | 1 +
 .../main/java/org/apache/kafka/streams/kstream/KStream.java    | 3 +++
 .../kafka/streams/kstream/internals/PrintForeachAction.java    | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/docs/streams/developer-guide/dsl-api.html b/docs/streams/developer-guide/dsl-api.html
index f61f052d5d052..01931c803ee3c 100644
--- a/docs/streams/developer-guide/dsl-api.html
+++ b/docs/streams/developer-guide/dsl-api.html
@@ -654,6 +654,7 @@ 
    Overviewdetails)
    
                             
    Calling print() is the same as calling foreach((key, value) -> System.out.println(key + ", " + value))
    
+                            
    print is mainly for debugging/testing purposes, and it will try to flush on each record print. Hence it should not be used for production usage if performance requirements are concerned.
    
                             
    KStream<byte[], String> stream = ...;
 // print to sysout
 stream.print();
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
index ae3b28a35ce0e..cf2ce75450efa 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
@@ -359,6 +359,9 @@ public interface KStream {
 
     /**
      * Print the records of this KStream using the options provided by {@link Printed}
+     * Note that this is mainly for debugging/testing purposes, and it will try to flush on each record print.
+     * It SHOULD NOT be used for production usage if performance requirements are concerned.
+     *
      * @param printed options for printing
      */
     void print(final Printed printed);
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java
index 174319fec4bd4..861dfd3b84e1e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/PrintForeachAction.java
@@ -51,6 +51,9 @@ public class PrintForeachAction implements ForeachAction {
     public void apply(final K key, final V value) {
         final String data = String.format("[%s]: %s", label, mapper.apply(key, value));
         printWriter.println(data);
+        if (!closable) {
+            printWriter.flush();
+        }
     }
 
     public void close() {

From 7299e18369999ba2ff9485c1256410c569e35379 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Sat, 1 Sep 2018 11:10:03 -0700
Subject: [PATCH 0774/1847] MINOR: Pass `--continue` to gradle in jenkins.sh

This ensures that the whole test suite is executed even
if there are failures. It currently stops at a module
boundary if there are any failures. There is a discussion
to change the gradle default to stop after the first test failure:

https://github.com/gradle/gradle/issues/6513

`--continue` is recommended for CI in that discussion.

Author: Ismael Juma 

Reviewers: Dong Lin 

Closes #5599 from ijuma/jenkins-script-continue
---
 jenkins.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins.sh b/jenkins.sh
index 8fff55d9d197e..43973aca6ac95 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -17,4 +17,4 @@
 # This script is used for verifying changes in Jenkins. In order to provide faster feedback, the tasks are ordered so
 # that faster tasks are executed in every module before slower tasks (if possible). For example, the unit tests for all
 # the modules are executed before the integration tests.
-./gradlew clean compileJava compileScala compileTestJava compileTestScala spotlessScalaCheck checkstyleMain checkstyleTest findbugsMain unitTest rat integrationTest --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed "$@"
+./gradlew clean compileJava compileScala compileTestJava compileTestScala spotlessScalaCheck checkstyleMain checkstyleTest findbugsMain unitTest rat integrationTest --no-daemon --continue -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed "$@"

From 6a1bb547c0c1f8e9f021e89958234f92f8a9014f Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Mon, 3 Sep 2018 07:21:20 -0700
Subject: [PATCH 0775/1847] KAFKA-7372: Upgrade Jetty for preliminary Java 11
 and TLS 1.3 support (#5600)

"Jetty 9.4.12 includes compatibility for JDK 11. Additionally, TLS 1.3 support has been implemented. While full functionality for new JDK features is not yet supported, this release has been built and tested for compatibility with the latest releases from Oracle."

http://dev.eclipse.org/mhonarc/lists/jetty-announce/msg00124.html

Reviewers: Rajini Sivaram 
---
 docs/upgrade.html          | 2 ++
 gradle/dependencies.gradle | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/upgrade.html b/docs/upgrade.html
index 3b7ee85345404..c8cd716be0bf1 100644
--- a/docs/upgrade.html
+++ b/docs/upgrade.html
@@ -43,6 +43,8 @@ 
    Upgrading from 0.8.x, 0.9.x, 0.1
 
 
    Notable changes in 2.1.0
    
 
      
+    
    • Jetty has been upgraded to 9.4.12, which excludes TLS_RSA_* ciphers by default because they do not support forward
+        secrecy, see https://github.com/eclipse/jetty.project/issues/2807 for more information.
      • 
     
    • Unclean leader election is automatically enabled by the controller when unclean.leader.election.enable config is dynamically updated by using per-topic config override.
      • 
 
    
 
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index 80af4d80996dd..0cdba7bc2b0a3 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -54,7 +54,7 @@ versions += [
   bcpkix: "1.59",
   easymock: "3.6",
   jackson: "2.9.6",
-  jetty: "9.4.11.v20180605",
+  jetty: "9.4.12.v20180830",
   jersey: "2.27",
   jmh: "1.21",
   log4j: "1.2.17",

From cb484666f2474747538585ed3b986aed5b02830d Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Mon, 3 Sep 2018 21:19:48 +0530
Subject: [PATCH 0776/1847] MINOR: Fix Transient test failure
 SslTransportLayerTest.testNetworkThreadTimeRecorded (#5597)

---
 .../java/org/apache/kafka/common/network/NioEchoServer.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java
index bd212fdc1720c..7996e5902c3dd 100644
--- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java
+++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java
@@ -95,7 +95,7 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco
         if (channelBuilder == null)
             channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, securityProtocol, config, credentialCache, tokenCache);
         this.metrics = new Metrics();
-        this.selector = new Selector(5000, failedAuthenticationDelayMs, metrics, time, "MetricGroup", channelBuilder, new LogContext());
+        this.selector = new Selector(10000, failedAuthenticationDelayMs, metrics, time, "MetricGroup", channelBuilder, new LogContext());
         acceptorThread = new AcceptorThread();
     }
 

From a8eac1b8149da20c797d42f426178b31032b76a0 Mon Sep 17 00:00:00 2001
From: Ismael Juma 
Date: Mon, 3 Sep 2018 10:21:48 -0700
Subject: [PATCH 0777/1847] MINOR: Use annotationProcessor instead of compile
 for JMH annotation processor

This fixes the following Gradle warning:

> The following annotation processors were detected on the compile classpath:
> 'org.openjdk.jmh.generators.BenchmarkProcessor'. Detecting annotation processors
> on the compile classpath is deprecated and Gradle 5.0 will ignore them. Please add
> them to the annotation processor path instead. If you did not intend to use annotation
> processors, you can use the '-proc:none' compiler argument to ignore them.

With this change, the warning went away and `./jmh-benchmarks/jmh.sh` continues
to work.

Author: Ismael Juma 

Reviewers: Dong Lin 

Closes #5602 from ijuma/annotation-processor
---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index b13e94856dfb2..9ebdfe9633e5d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1198,7 +1198,7 @@ project(':jmh-benchmarks') {
     compile project(':clients')
     compile project(':streams')
     compile libs.jmhCore
-    compile libs.jmhGeneratorAnnProcess
+    annotationProcessor libs.jmhGeneratorAnnProcess
     compile libs.jmhCoreBenchmarks
   }
 

From 62ea10a5c17b6a8d3ebed26cdba813647585dc20 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Tue, 4 Sep 2018 17:59:50 +0530
Subject: [PATCH 0778/1847] MINOR: Update README to specify Gradle 4.6 as the
 minimum required version (#5606)

#5602 uses `annotationProcessor`, which was introduced in Gradle 4.6.

Reviewers: Ismael Juma 
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 18cb03e988048..228581426b968 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ See our [web site](http://kafka.apache.org) for details on the project.
 
 You need to have [Gradle](http://www.gradle.org/installation) and [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) installed.
 
-Kafka requires Gradle 4.5 or higher.
+Kafka requires Gradle 4.6 or higher.
 
 Java 8 should be used for building in order to support both Java 8 and Java 10 at runtime.
 

From 4a46a353482116ed5502737f7809bc6c0d37bf36 Mon Sep 17 00:00:00 2001
From: Rajini Sivaram 
Date: Tue, 4 Sep 2018 21:24:06 +0100
Subject: [PATCH 0779/1847] MINOR: Tidy up pattern type comparisons, remove
 unused producer-id (#5593)

Reviewers: Ismael Juma , Manikumar Reddy 
---
 .../resource/ResourcePatternFilter.java       |  2 +-
 .../main/scala/kafka/admin/AclCommand.scala   |  2 +-
 .../scala/kafka/security/SecurityUtils.scala  |  1 +
 .../scala/kafka/security/auth/Resource.scala  | 11 +++----
 .../main/scala/kafka/server/KafkaApis.scala   |  3 +-
 core/src/main/scala/kafka/zk/ZkData.scala     |  4 +--
 .../unit/kafka/admin/AclCommandTest.scala     | 32 ++++++++++++++++++-
 7 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/resource/ResourcePatternFilter.java b/clients/src/main/java/org/apache/kafka/common/resource/ResourcePatternFilter.java
index 83f5c88c90ddb..6f511c9a345d6 100644
--- a/clients/src/main/java/org/apache/kafka/common/resource/ResourcePatternFilter.java
+++ b/clients/src/main/java/org/apache/kafka/common/resource/ResourcePatternFilter.java
@@ -139,7 +139,7 @@ public String findIndefiniteField() {
         if (name == null)
             return "Resource name is NULL.";
         if (patternType == PatternType.MATCH)
-            return "Resource pattern type is ANY.";
+            return "Resource pattern type is MATCH.";
         if (patternType == PatternType.UNKNOWN)
             return "Resource pattern type is UNKNOWN.";
         return null;
diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index e86e1a3e14d20..31e6c53dc1142 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -88,7 +88,7 @@ object AclCommand extends Logging {
 
   private def addAcl(opts: AclCommandOptions) {
     val patternType: PatternType = opts.options.valueOf(opts.resourcePatternType)
-    if (patternType == PatternType.MATCH || patternType == PatternType.ANY)
+    if (!patternType.isSpecific)
       CommandLineUtils.printUsageAndDie(opts.parser, s"A '--resource-pattern-type' value of '$patternType' is not valid when adding acls.")
 
     withAuthorizer(opts) { authorizer =>
diff --git a/core/src/main/scala/kafka/security/SecurityUtils.scala b/core/src/main/scala/kafka/security/SecurityUtils.scala
index 74bd4043edb9f..5d42871f66f85 100644
--- a/core/src/main/scala/kafka/security/SecurityUtils.scala
+++ b/core/src/main/scala/kafka/security/SecurityUtils.scala
@@ -50,4 +50,5 @@ object SecurityUtils {
     new AclBinding(resourcePattern, entry)
   }
 
+  def isClusterResource(name: String): Boolean = name.equals(Resource.ClusterResourceName)
 }
diff --git a/core/src/main/scala/kafka/security/auth/Resource.scala b/core/src/main/scala/kafka/security/auth/Resource.scala
index c4755961f94a5..b062ce295eb1b 100644
--- a/core/src/main/scala/kafka/security/auth/Resource.scala
+++ b/core/src/main/scala/kafka/security/auth/Resource.scala
@@ -23,9 +23,11 @@ object Resource {
   val Separator = ":"
   val ClusterResourceName = "kafka-cluster"
   val ClusterResource = Resource(Cluster, Resource.ClusterResourceName, PatternType.LITERAL)
-  val ProducerIdResourceName = "producer-id"
   val WildCardResource = "*"
 
+  @deprecated("This resource name is not used by Kafka and will be removed in a future release", since = "2.1")
+  val ProducerIdResourceName = "producer-id" // This is not used since we don't have a producer id resource
+
   def fromString(str: String): Resource = {
     ResourceType.values.find(resourceType => str.startsWith(resourceType.name + Separator)) match {
       case None => throw new KafkaException("Invalid resource string: '" + str + "'")
@@ -53,11 +55,8 @@ object Resource {
  */
 case class Resource(resourceType: ResourceType, name: String, patternType: PatternType) {
 
-  if (patternType == PatternType.MATCH || patternType == PatternType.ANY)
-    throw new IllegalArgumentException("patternType must not be " + patternType)
-
-  if (patternType == PatternType.UNKNOWN)
-    throw new IllegalArgumentException("patternType must not be UNKNOWN")
+  if (!patternType.isSpecific)
+    throw new IllegalArgumentException(s"patternType must not be $patternType")
 
   /**
     * Create an instance of this class with the provided parameters.
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 096ff388bdd46..3a81b89f61a9e 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -1950,8 +1950,7 @@ class KafkaApis(val requestChannel: RequestChannel,
           SecurityUtils.convertToResourceAndAcl(aclCreation.acl.toFilter) match {
             case Left(apiError) => new AclCreationResponse(apiError)
             case Right((resource, acl)) => try {
-                if (resource.resourceType.equals(Cluster) &&
-                    !resource.name.equals(Resource.ClusterResourceName))
+                if (resource.resourceType.equals(Cluster) && !SecurityUtils.isClusterResource(resource.name))
                   throw new InvalidRequestException("The only valid name for the CLUSTER resource is " +
                       Resource.ClusterResourceName)
                 if (resource.name.isEmpty)
diff --git a/core/src/main/scala/kafka/zk/ZkData.scala b/core/src/main/scala/kafka/zk/ZkData.scala
index 760bd67299dbf..ed687cb30f9f7 100644
--- a/core/src/main/scala/kafka/zk/ZkData.scala
+++ b/core/src/main/scala/kafka/zk/ZkData.scala
@@ -497,9 +497,7 @@ sealed trait ZkAclStore {
 
 object ZkAclStore {
   private val storesByType: Map[PatternType, ZkAclStore] = PatternType.values
-    .filter(patternType => patternType != PatternType.MATCH)
-    .filter(patternType => patternType != PatternType.ANY)
-    .filter(patternType => patternType != PatternType.UNKNOWN)
+    .filter(_.isSpecific)
     .map(patternType => (patternType, create(patternType)))
     .toMap
 
diff --git a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
index 26ae073501307..05f61897e6a70 100644
--- a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
@@ -21,8 +21,9 @@ import java.util.Properties
 import kafka.admin.AclCommand.AclCommandOptions
 import kafka.security.auth._
 import kafka.server.KafkaConfig
-import kafka.utils.{Logging, TestUtils}
+import kafka.utils.{Exit, Logging, TestUtils}
 import kafka.zk.ZooKeeperTestHarness
+import org.apache.kafka.common.resource.PatternType
 import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED}
 import org.apache.kafka.common.security.auth.KafkaPrincipal
 import org.junit.{Before, Test}
@@ -158,6 +159,35 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     AclCommand.withAuthorizer(new AclCommandOptions(args))(null)
   }
 
+  @Test
+  def testPatternTypes() {
+    Exit.setExitProcedure { (status, _) =>
+      if (status == 1)
+        throw new RuntimeException("Exiting command")
+      else
+        throw new AssertionError(s"Unexpected exit with status $status")
+    }
+    def verifyPatternType(cmd: Array[String], isValid: Boolean): Unit = {
+      if (isValid)
+        AclCommand.main(cmd)
+      else
+        intercept[RuntimeException](AclCommand.main(cmd))
+    }
+    try {
+      PatternType.values.foreach { patternType =>
+        val addCmd = zkArgs ++ Array("--allow-principal", principal.toString, "--producer", "--topic", "Test",
+          "--add", "--resource-pattern-type", patternType.toString)
+        verifyPatternType(addCmd, isValid = patternType.isSpecific)
+        val listCmd = zkArgs ++ Array("--topic", "Test", "--list", "--resource-pattern-type", patternType.toString)
+        verifyPatternType(listCmd, isValid = patternType != PatternType.UNKNOWN)
+        val removeCmd = zkArgs ++ Array("--topic", "Test", "--force", "--remove", "--resource-pattern-type", patternType.toString)
+        verifyPatternType(removeCmd, isValid = patternType != PatternType.UNKNOWN)
+      }
+    } finally {
+      Exit.resetExitProcedure()
+    }
+  }
+
   private def testRemove(resources: Set[Resource], resourceCmd: Array[String], brokerProps: Properties) {
     for (resource <- resources) {
       AclCommand.main(zkArgs ++ resourceCmd :+ "--remove" :+ "--force")

From 79a608b286fd6271b841d3cc7997bf69227de912 Mon Sep 17 00:00:00 2001
From: huxihx 
Date: Tue, 4 Sep 2018 13:55:49 -0700
Subject: [PATCH 0780/1847] KAFKA-7211; MM should handle TimeoutException in
 commitSync

With KIP-266 introduced, MirrorMaker should handle TimeoutException thrown in commitSync(). Besides, MM should only commit offsets for existsing topics.

Author: huxihx 

Reviewers: Dong Lin 

Closes #5492 from huxihx/KAFKA-7211
---
 .../main/scala/kafka/tools/MirrorMaker.scala  | 67 +++++++++++++------
 .../tools/MirrorMakerIntegrationTest.scala    | 30 +++++++++
 2 files changed, 76 insertions(+), 21 deletions(-)

diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index d55d96bd65bc8..b6fd918032c4f 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -34,17 +34,18 @@ import org.apache.kafka.clients.producer.internals.ErrorLoggingCallback
 import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord, RecordMetadata}
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer}
-import org.apache.kafka.common.utils.Utils
-import org.apache.kafka.common.errors.WakeupException
+import org.apache.kafka.common.utils.{Time, Utils}
+import org.apache.kafka.common.errors.{TimeoutException, WakeupException}
 import org.apache.kafka.common.record.RecordBatch
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
+import scala.util.{Failure, Success, Try}
 import scala.util.control.ControlThrowable
 
 /**
  * The mirror maker has the following architecture:
- * - There are N mirror maker thread shares one ZookeeperConsumerConnector and each owns a Kafka stream.
+ * - There are N mirror maker thread, each of which is equipped with a separate KafkaConsumer instance.
  * - All the mirror maker threads share one producer.
  * - Each mirror maker thread periodically flushes the producer and then commits all offsets.
  *
@@ -70,6 +71,8 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
   private var offsetCommitIntervalMs = 0
   private var abortOnSendFailure: Boolean = true
   @volatile private var exitingOnSendFailure: Boolean = false
+  private var lastSuccessfulCommitTime = -1L
+  private val time = Time.SYSTEM
 
   // If a message send failed after retries are exhausted. The offset of the messages will also be removed from
   // the unacked offset list to avoid offset commit being stuck on that offset. In this case, the offset of that
@@ -268,24 +271,45 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
     consumers.map(consumer => new ConsumerWrapper(consumer, customRebalanceListener, whitelist))
   }
 
-  def commitOffsets(consumerWrapper: ConsumerWrapper) {
+  def commitOffsets(consumerWrapper: ConsumerWrapper): Unit = {
     if (!exitingOnSendFailure) {
-      trace("Committing offsets.")
-      try {
-        consumerWrapper.commit()
-      } catch {
-        case e: WakeupException =>
-          // we only call wakeup() once to close the consumer,
-          // so if we catch it in commit we can safely retry
-          // and re-throw to break the loop
+      var retry = 0
+      var retryNeeded = true
+      while (retryNeeded) {
+        trace("Committing offsets.")
+        try {
           consumerWrapper.commit()
-          throw e
+          lastSuccessfulCommitTime = time.milliseconds
+          retryNeeded = false
+        } catch {
+          case e: WakeupException =>
+            // we only call wakeup() once to close the consumer,
+            // so if we catch it in commit we can safely retry
+            // and re-throw to break the loop
+            commitOffsets(consumerWrapper)
+            throw e
+
+          case _: TimeoutException =>
+            Try(consumerWrapper.consumer.listTopics) match {
+              case Success(visibleTopics) =>
+                consumerWrapper.offsets.retain((tp, _) => visibleTopics.containsKey(tp.topic))
+              case Failure(e) =>
+                warn("Failed to list all authorized topics after committing offsets timed out: ", e)
+            }
 
-        case _: CommitFailedException =>
-          warn("Failed to commit offsets because the consumer group has rebalanced and assigned partitions to " +
-            "another instance. If you see this regularly, it could indicate that you need to either increase " +
-            s"the consumer's ${ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG} or reduce the number of records " +
-            s"handled on each iteration with ${ConsumerConfig.MAX_POLL_RECORDS_CONFIG}")
+            retry += 1
+            warn("Failed to commit offsets because the offset commit request processing can not be completed in time. " +
+              s"If you see this regularly, it could indicate that you need to increase the consumer's ${ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG} " +
+              s"Last successful offset commit timestamp=$lastSuccessfulCommitTime, retry count=$retry")
+            Thread.sleep(100)
+
+          case _: CommitFailedException =>
+            retryNeeded = false
+            warn("Failed to commit offsets because the consumer group has rebalanced and assigned partitions to " +
+              "another instance. If you see this regularly, it could indicate that you need to either increase " +
+              s"the consumer's ${ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG} or reduce the number of records " +
+              s"handled on each iteration with ${ConsumerConfig.MAX_POLL_RECORDS_CONFIG}")
+        }
       }
     } else {
       info("Exiting on send failure, skip committing offsets.")
@@ -423,14 +447,15 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
   }
 
   // Visible for testing
-  private[tools] class ConsumerWrapper(consumer: Consumer[Array[Byte], Array[Byte]],
+  private[tools] class ConsumerWrapper(private[tools] val consumer: Consumer[Array[Byte], Array[Byte]],
                                        customRebalanceListener: Option[ConsumerRebalanceListener],
                                        whitelistOpt: Option[String]) {
     val regex = whitelistOpt.getOrElse(throw new IllegalArgumentException("New consumer only supports whitelist."))
     var recordIter: java.util.Iterator[ConsumerRecord[Array[Byte], Array[Byte]]] = null
 
     // We manually maintain the consumed offsets for historical reasons and it could be simplified
-    private val offsets = new HashMap[TopicPartition, Long]()
+    // Visible for testing
+    private[tools] val offsets = new HashMap[TopicPartition, Long]()
 
     def init() {
       debug("Initiating consumer")
@@ -474,7 +499,7 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
     }
 
     def commit() {
-      consumer.commitSync(offsets.map { case (tp, offset) =>  (tp, new OffsetAndMetadata(offset, ""))}.asJava)
+      consumer.commitSync(offsets.map { case (tp, offset) => (tp, new OffsetAndMetadata(offset)) }.asJava)
       offsets.clear()
     }
   }
diff --git a/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala b/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala
index 0a178195bef2a..7212b3b351e9a 100644
--- a/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/tools/MirrorMakerIntegrationTest.scala
@@ -24,15 +24,45 @@ import kafka.tools.MirrorMaker.{ConsumerWrapper, MirrorMakerProducer}
 import kafka.utils.TestUtils
 import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
 import org.apache.kafka.clients.producer.{ProducerConfig, ProducerRecord}
+import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.TimeoutException
 import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer}
 import org.junit.Test
+import org.junit.Assert._
 
 class MirrorMakerIntegrationTest extends KafkaServerTestHarness {
 
   override def generateConfigs: Seq[KafkaConfig] =
     TestUtils.createBrokerConfigs(1, zkConnect).map(KafkaConfig.fromProps(_, new Properties()))
 
+  @Test(expected = classOf[TimeoutException])
+  def testCommitOffsetsThrowTimeoutException(): Unit = {
+    val consumerProps = new Properties
+    consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group")
+    consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    consumerProps.put(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "1")
+    val consumer = new KafkaConsumer(consumerProps, new ByteArrayDeserializer, new ByteArrayDeserializer)
+    val mirrorMakerConsumer = new ConsumerWrapper(consumer, None, whitelistOpt = Some("any"))
+    mirrorMakerConsumer.offsets.put(new TopicPartition("test", 0), 0L)
+    mirrorMakerConsumer.commit()
+  }
+
+  @Test
+  def testCommitOffsetsRemoveNonExistentTopics(): Unit = {
+    val consumerProps = new Properties
+    consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group")
+    consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
+    consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
+    consumerProps.put(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "2000")
+    val consumer = new KafkaConsumer(consumerProps, new ByteArrayDeserializer, new ByteArrayDeserializer)
+    val mirrorMakerConsumer = new ConsumerWrapper(consumer, None, whitelistOpt = Some("any"))
+    mirrorMakerConsumer.offsets.put(new TopicPartition("nonexistent-topic1", 0), 0L)
+    mirrorMakerConsumer.offsets.put(new TopicPartition("nonexistent-topic2", 0), 0L)
+    MirrorMaker.commitOffsets(mirrorMakerConsumer)
+    assertTrue("Offsets for non-existent topics should be removed", mirrorMakerConsumer.offsets.isEmpty)
+  }
+
   @Test
   def testCommaSeparatedRegex(): Unit = {
     val topic = "new-topic"

From 847780e5a5f376fa2ce8705f483bfd33b319b83d Mon Sep 17 00:00:00 2001
From: Kevin Lafferty 
Date: Wed, 5 Sep 2018 20:15:25 -0700
Subject: [PATCH 0781/1847] KAFKA-7353: Connect logs 'this' for anonymous inner
 classes

Replace 'this' reference in anonymous inner class logs to out class's 'this'

Author: Kevin Lafferty 

Reviewers: Randall Hauch , Arjun Satish , Ewen Cheslack-Postava 

Closes #5583 from kevin-laff/connect_logging
---
 .../apache/kafka/connect/runtime/WorkerConnector.java  |  2 +-
 .../apache/kafka/connect/runtime/WorkerSinkTask.java   |  2 +-
 .../apache/kafka/connect/runtime/WorkerSourceTask.java | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
index 611e196d9de9d..55d4860b2e66b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
@@ -89,7 +89,7 @@ public void requestTaskReconfiguration() {
 
                 @Override
                 public void raiseError(Exception e) {
-                    log.error("{} Connector raised an error", this, e);
+                    log.error("{} Connector raised an error", WorkerConnector.this, e);
                     onFailure(e);
                     ctx.raiseError(e);
                 }
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 692331ed13f71..39e0c6d53f645 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -649,7 +649,7 @@ public void onPartitionsAssigned(Collection partitions) {
                 long pos = consumer.position(tp);
                 lastCommittedOffsets.put(tp, new OffsetAndMetadata(pos));
                 currentOffsets.put(tp, new OffsetAndMetadata(pos));
-                log.debug("{} Assigned topic partition {} with offset {}", this, tp, pos);
+                log.debug("{} Assigned topic partition {} with offset {}", WorkerSinkTask.this, tp, pos);
             }
             sinkTaskMetricsGroup.assignedOffsets(currentOffsets);
 
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index 70d0cf9d7aea3..623a210e0e27b 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -326,11 +326,11 @@ public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                                     // timeouts, callbacks with exceptions should never be invoked in practice. If the
                                     // user overrode these settings, the best we can do is notify them of the failure via
                                     // logging.
-                                    log.error("{} failed to send record to {}: {}", this, topic, e);
-                                    log.debug("{} Failed record: {}", this, preTransformRecord);
+                                    log.error("{} failed to send record to {}: {}", WorkerSourceTask.this, topic, e);
+                                    log.debug("{} Failed record: {}", WorkerSourceTask.this, preTransformRecord);
                                 } else {
                                     log.trace("{} Wrote record successfully: topic {} partition {} offset {}",
-                                            this,
+                                            WorkerSourceTask.this,
                                             recordMetadata.topic(), recordMetadata.partition(),
                                             recordMetadata.offset());
                                     commitTaskRecord(preTransformRecord);
@@ -454,9 +454,9 @@ public boolean commitOffsets() {
             @Override
             public void onCompletion(Throwable error, Void result) {
                 if (error != null) {
-                    log.error("{} Failed to flush offsets to storage: ", this, error);
+                    log.error("{} Failed to flush offsets to storage: ", WorkerSourceTask.this, error);
                 } else {
-                    log.trace("{} Finished flushing offsets to storage", this);
+                    log.trace("{} Finished flushing offsets to storage", WorkerSourceTask.this);
                 }
             }
         });

From 297fb396a0038addea06a76fd4ab9a451eb7562e Mon Sep 17 00:00:00 2001
From: "Zhanxiang (Patrick) Huang" 
Date: Fri, 7 Sep 2018 14:17:49 -0700
Subject: [PATCH 0782/1847] KAFKA-6082; Fence zookeeper updates with controller
 epoch zkVersion

This PR aims to enforce that the controller can only update zookeeper states after checking the controller epoch zkVersion. The check and zookeeper state updates are wrapped in the zookeeper multi() operations to ensure that they are done atomically. This PR is necessary to resolve issues related to multiple controllers (i.e. old controller updates zookeeper states before resignation, which is possible during controller failover based on the single threaded event queue model we have)

This PR includes the following changes:
- Add MultiOp request and response in ZookeeperClient
- Ensure all zookeeper updates done by controller are protected by checking the current controller epoch zkVersion
- Modify test cases in KafkaZkClientTest to test mismatch controller epoch zkVersion

Tests Done:
- Unit tests (with updated tests to test mismatch controller epoch zkVersion)
- Existing integration tests

Author: Zhanxiang (Patrick) Huang 

Reviewers: Jun Rao , Dong Lin , Manikumar Reddy O 

Closes #5101 from hzxa21/KAFKA-6082
---
 .../main/scala/kafka/cluster/Partition.scala  |   2 +-
 .../kafka/controller/ControllerContext.scala  |   4 +-
 .../controller/ControllerEventManager.scala   |  10 +-
 .../kafka/controller/KafkaController.scala    | 161 ++++++------
 .../controller/PartitionStateMachine.scala    |  11 +-
 .../controller/ReplicaStateMachine.scala      |   6 +-
 .../controller/TopicDeletionManager.scala     |   8 +-
 .../scala/kafka/server/ReplicaManager.scala   |   2 +-
 .../main/scala/kafka/zk/KafkaZkClient.scala   | 232 +++++++++++++-----
 .../kafka/zookeeper/ZooKeeperClient.scala     | 134 ++++++----
 .../admin/ReassignPartitionsClusterTest.scala |   4 +-
 .../ControllerEventManagerTest.scala          |   2 +-
 .../ControllerIntegrationTest.scala           | 153 ++++++++++--
 .../PartitionStateMachineTest.scala           |  16 +-
 .../controller/ReplicaStateMachineTest.scala  |   2 +-
 .../unit/kafka/utils/LogCaptureAppender.scala |  66 +++++
 .../kafka/utils/ReplicationUtilsTest.scala    |   4 +-
 .../scala/unit/kafka/utils/TestUtils.scala    |   4 +-
 .../unit/kafka/zk/KafkaZkClientTest.scala     | 157 +++++++-----
 19 files changed, 680 insertions(+), 298 deletions(-)
 create mode 100644 core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala

diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 22c1508bb8ca0..d76d6d06b5bdd 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -73,7 +73,7 @@ class Partition(val topic: String,
    * the controller sends it a start replica command containing the leader for each partition that the broker hosts.
    * In addition to the leader, the controller can also send the epoch of the controller that elected the leader for
    * each partition. */
-  private var controllerEpoch: Int = KafkaController.InitialControllerEpoch - 1
+  private var controllerEpoch: Int = KafkaController.InitialControllerEpoch
   this.logIdent = s"[Partition $topicPartition broker=$localBrokerId] "
 
   private def isReplicaLocal(replicaId: Int) : Boolean = replicaId == localBrokerId || replicaId == Request.FutureLocalReplicaId
diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala
index f4671cfaa1a63..20c3de0c1e549 100644
--- a/core/src/main/scala/kafka/controller/ControllerContext.scala
+++ b/core/src/main/scala/kafka/controller/ControllerContext.scala
@@ -28,8 +28,8 @@ class ControllerContext {
   var controllerChannelManager: ControllerChannelManager = null
 
   var shuttingDownBrokerIds: mutable.Set[Int] = mutable.Set.empty
-  var epoch: Int = KafkaController.InitialControllerEpoch - 1
-  var epochZkVersion: Int = KafkaController.InitialControllerEpochZkVersion - 1
+  var epoch: Int = KafkaController.InitialControllerEpoch
+  var epochZkVersion: Int = KafkaController.InitialControllerEpochZkVersion
   var allTopics: Set[String] = Set.empty
   private var partitionReplicaAssignmentUnderlying: mutable.Map[String, mutable.Map[Int, Seq[Int]]] = mutable.Map.empty
   val partitionLeadershipInfo: mutable.Map[TopicPartition, LeaderIsrAndControllerEpoch] = mutable.Map.empty
diff --git a/core/src/main/scala/kafka/controller/ControllerEventManager.scala b/core/src/main/scala/kafka/controller/ControllerEventManager.scala
index 13967e029ed4e..c93e9e79ec21f 100644
--- a/core/src/main/scala/kafka/controller/ControllerEventManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerEventManager.scala
@@ -24,6 +24,7 @@ import com.yammer.metrics.core.Gauge
 import kafka.metrics.{KafkaMetricsGroup, KafkaTimer}
 import kafka.utils.CoreUtils.inLock
 import kafka.utils.ShutdownableThread
+import org.apache.kafka.common.errors.ControllerMovedException
 import org.apache.kafka.common.utils.Time
 
 import scala.collection._
@@ -32,12 +33,14 @@ object ControllerEventManager {
   val ControllerEventThreadName = "controller-event-thread"
 }
 class ControllerEventManager(controllerId: Int, rateAndTimeMetrics: Map[ControllerState, KafkaTimer],
-                             eventProcessedListener: ControllerEvent => Unit) extends KafkaMetricsGroup {
+                             eventProcessedListener: ControllerEvent => Unit,
+                             controllerMovedListener: () => Unit) extends KafkaMetricsGroup {
 
   @volatile private var _state: ControllerState = ControllerState.Idle
   private val putLock = new ReentrantLock()
   private val queue = new LinkedBlockingQueue[ControllerEvent]
-  private val thread = new ControllerEventThread(ControllerEventManager.ControllerEventThreadName)
+  // Visible for test
+  private[controller] val thread = new ControllerEventThread(ControllerEventManager.ControllerEventThreadName)
   private val time = Time.SYSTEM
 
   private val eventQueueTimeHist = newHistogram("EventQueueTimeMs")
@@ -86,6 +89,9 @@ class ControllerEventManager(controllerId: Int, rateAndTimeMetrics: Map[Controll
               controllerEvent.process()
             }
           } catch {
+            case e: ControllerMovedException =>
+              info(s"Controller moved to another broker when processing $controllerEvent.", e)
+              controllerMovedListener()
             case e: Throwable => error(s"Error processing event $controllerEvent", e)
           }
 
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 286768f595bf5..379e66da9e0d0 100644
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -35,14 +35,14 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors}
 import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, LeaderAndIsrResponse, StopReplicaResponse}
 import org.apache.kafka.common.utils.Time
 import org.apache.zookeeper.KeeperException
-import org.apache.zookeeper.KeeperException.{Code, NodeExistsException}
+import org.apache.zookeeper.KeeperException.Code
 
 import scala.collection._
 import scala.util.Try
 
 object KafkaController extends Logging {
-  val InitialControllerEpoch = 1
-  val InitialControllerEpochZkVersion = 1
+  val InitialControllerEpoch = 0
+  val InitialControllerEpochZkVersion = 0
 
   /**
    * ControllerEventThread will shutdown once it sees this event
@@ -52,6 +52,12 @@ object KafkaController extends Logging {
     override def process(): Unit = ()
   }
 
+  // Used only by test
+  private[controller] case class AwaitOnLatch(latch: CountDownLatch) extends ControllerEvent {
+    override def state: ControllerState = ControllerState.ControllerChange
+    override def process(): Unit = latch.await()
+  }
+
 }
 
 class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Time, metrics: Metrics, initialBrokerInfo: BrokerInfo,
@@ -70,7 +76,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
 
   // visible for testing
   private[controller] val eventManager = new ControllerEventManager(config.brokerId,
-    controllerContext.stats.rateAndTimeMetrics, _ => updateMetrics())
+    controllerContext.stats.rateAndTimeMetrics, _ => updateMetrics(), () => maybeResign())
 
   val topicDeletionManager = new TopicDeletionManager(this, eventManager, zkClient)
   private val brokerRequestBatch = new ControllerBrokerRequestBatch(this, stateChangeLogger)
@@ -214,21 +220,15 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
   /**
    * This callback is invoked by the zookeeper leader elector on electing the current broker as the new controller.
    * It does the following things on the become-controller state change -
-   * 1. Registers controller epoch changed listener
-   * 2. Increments the controller epoch
-   * 3. Initializes the controller's context object that holds cache objects for current topics, live brokers and
+   * 1. Initializes the controller's context object that holds cache objects for current topics, live brokers and
    *    leaders for all existing partitions.
-   * 4. Starts the controller's channel manager
-   * 5. Starts the replica state machine
-   * 6. Starts the partition state machine
+   * 2. Starts the controller's channel manager
+   * 3. Starts the replica state machine
+   * 4. Starts the partition state machine
    * If it encounters any unexpected exception/error while becoming controller, it resigns as the current controller.
    * This ensures another controller election will be triggered and there will always be an actively serving controller
    */
   private def onControllerFailover() {
-    info("Reading controller epoch from ZooKeeper")
-    readControllerEpochFromZooKeeper()
-    info("Incrementing controller epoch in ZooKeeper")
-    incrementControllerEpoch()
     info("Registering handlers")
 
     // before reading source of truth from zookeeper, register the listeners to get broker/topic callbacks
@@ -239,9 +239,9 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     nodeChangeHandlers.foreach(zkClient.registerZNodeChangeHandlerAndCheckExistence)
 
     info("Deleting log dir event notifications")
-    zkClient.deleteLogDirEventNotifications()
+    zkClient.deleteLogDirEventNotifications(controllerContext.epochZkVersion)
     info("Deleting isr change notifications")
-    zkClient.deleteIsrChangeNotifications()
+    zkClient.deleteIsrChangeNotifications(controllerContext.epochZkVersion)
     info("Initializing controller context")
     initializeControllerContext()
     info("Fetching topic deletions in progress")
@@ -599,6 +599,9 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
               topicDeletionManager.markTopicIneligibleForDeletion(Set(topic))
               onPartitionReassignment(tp, reassignedPartitionContext)
             } catch {
+              case e: ControllerMovedException =>
+                error(s"Error completing reassignment of partition $tp because controller has moved to another broker", e)
+                throw e
               case e: Throwable =>
                 error(s"Error completing reassignment of partition $tp", e)
                 // remove the partition from the admin path to unblock the admin client
@@ -619,41 +622,15 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     try {
       partitionStateMachine.handleStateChanges(partitions.toSeq, OnlinePartition, Option(PreferredReplicaPartitionLeaderElectionStrategy))
     } catch {
+      case e: ControllerMovedException =>
+        error(s"Error completing preferred replica leader election for partitions ${partitions.mkString(",")} because controller has moved to another broker.", e)
+        throw e
       case e: Throwable => error(s"Error completing preferred replica leader election for partitions ${partitions.mkString(",")}", e)
     } finally {
       removePartitionsFromPreferredReplicaElection(partitions, isTriggeredByAutoRebalance)
     }
   }
 
-  private def incrementControllerEpoch(): Unit = {
-    val newControllerEpoch = controllerContext.epoch + 1
-    val setDataResponse = zkClient.setControllerEpochRaw(newControllerEpoch, controllerContext.epochZkVersion)
-    setDataResponse.resultCode match {
-      case Code.OK =>
-        controllerContext.epochZkVersion = setDataResponse.stat.getVersion
-        controllerContext.epoch = newControllerEpoch
-      case Code.NONODE =>
-        // if path doesn't exist, this is the first controller whose epoch should be 1
-        // the following call can still fail if another controller gets elected between checking if the path exists and
-        // trying to create the controller epoch path
-        val createResponse = zkClient.createControllerEpochRaw(KafkaController.InitialControllerEpoch)
-        createResponse.resultCode match {
-          case Code.OK =>
-            controllerContext.epoch = KafkaController.InitialControllerEpoch
-            controllerContext.epochZkVersion = KafkaController.InitialControllerEpochZkVersion
-          case Code.NODEEXISTS =>
-            throw new ControllerMovedException("Controller moved to another broker. Aborting controller startup procedure")
-          case _ =>
-            val exception = createResponse.resultException.get
-            error("Error while incrementing controller epoch", exception)
-            throw exception
-        }
-      case _ =>
-        throw new ControllerMovedException("Controller moved to another broker. Aborting controller startup procedure")
-    }
-    info(s"Epoch incremented to ${controllerContext.epoch}")
-  }
-
   private def initializeControllerContext() {
     // update controller cache with delete topic information
     controllerContext.liveBrokers = zkClient.getAllBrokersInCluster.toSet
@@ -783,7 +760,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
   private def updateAssignedReplicasForPartition(partition: TopicPartition,
                                                  replicas: Seq[Int]) {
     controllerContext.updatePartitionReplicaAssignment(partition, replicas)
-    val setDataResponse = zkClient.setTopicAssignmentRaw(partition.topic, controllerContext.partitionReplicaAssignmentForTopic(partition.topic))
+    val setDataResponse = zkClient.setTopicAssignmentRaw(partition.topic, controllerContext.partitionReplicaAssignmentForTopic(partition.topic), controllerContext.epochZkVersion)
     setDataResponse.resultCode match {
       case Code.OK =>
         info(s"Updated assigned replicas for partition $partition being reassigned to ${replicas.mkString(",")}")
@@ -844,16 +821,6 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     controllerContext.partitionsBeingReassigned.values.foreach(_.unregisterReassignIsrChangeHandler(zkClient))
   }
 
-  private def readControllerEpochFromZooKeeper() {
-    // initialize the controller epoch and zk version by reading from zookeeper
-    val epochAndStatOpt = zkClient.getControllerEpoch
-    epochAndStatOpt.foreach { case (epoch, stat) =>
-      controllerContext.epoch = epoch
-      controllerContext.epochZkVersion = stat.getVersion
-      info(s"Initialized controller epoch to ${controllerContext.epoch} and zk version ${controllerContext.epochZkVersion}")
-    }
-  }
-
   /**
    * Remove partition from partitions being reassigned in ZooKeeper and ControllerContext. If the partition reassignment
    * is complete (i.e. there is no other partition with a reassignment in progress), the reassign_partitions znode
@@ -874,12 +841,12 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     // write the new list to zookeeper
     if (updatedPartitionsBeingReassigned.isEmpty) {
       info(s"No more partitions need to be reassigned. Deleting zk path ${ReassignPartitionsZNode.path}")
-      zkClient.deletePartitionReassignment()
+      zkClient.deletePartitionReassignment(controllerContext.epochZkVersion)
       // Ensure we detect future reassignments
       eventManager.put(PartitionReassignment)
     } else {
       val reassignment = updatedPartitionsBeingReassigned.mapValues(_.newReplicas)
-      try zkClient.setOrCreatePartitionReassignment(reassignment)
+      try zkClient.setOrCreatePartitionReassignment(reassignment, controllerContext.epochZkVersion)
       catch {
         case e: KeeperException => throw new AdminOperationException(e)
       }
@@ -902,7 +869,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
       }
     }
     if (!isTriggeredByAutoRebalance) {
-      zkClient.deletePreferredReplicaElection()
+      zkClient.deletePreferredReplicaElection(controllerContext.epochZkVersion)
       // Ensure we detect future preferred replica leader elections
       eventManager.put(PreferredReplicaLeaderElection)
     }
@@ -955,7 +922,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
           val newLeaderAndIsr = leaderAndIsr.newEpochAndZkVersion
           // update the new leadership decision in zookeeper or retry
           val UpdateLeaderAndIsrResult(successfulUpdates, _, failedUpdates) =
-            zkClient.updateLeaderAndIsr(immutable.Map(partition -> newLeaderAndIsr), epoch)
+            zkClient.updateLeaderAndIsr(immutable.Map(partition -> newLeaderAndIsr), epoch, controllerContext.epochZkVersion)
           if (successfulUpdates.contains(partition)) {
             val finalLeaderAndIsr = successfulUpdates(partition)
             finalLeaderIsrAndControllerEpoch = Some(LeaderIsrAndControllerEpoch(finalLeaderAndIsr, epoch))
@@ -1204,13 +1171,32 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
   }
 
   private def triggerControllerMove(): Unit = {
-    onControllerResignation()
-    activeControllerId = -1
-    zkClient.deleteController()
+    activeControllerId = zkClient.getControllerId.getOrElse(-1)
+    if (!isActive) {
+      warn("Controller has already moved when trying to trigger controller movement")
+      return
+    }
+    try {
+      val expectedControllerEpochZkVersion = controllerContext.epochZkVersion
+      activeControllerId = -1
+      onControllerResignation()
+      zkClient.deleteController(expectedControllerEpochZkVersion)
+    } catch {
+      case _: ControllerMovedException =>
+        warn("Controller has already moved when trying to trigger controller movement")
+    }
+  }
+
+  private def maybeResign(): Unit = {
+    val wasActiveBeforeChange = isActive
+    zkClient.registerZNodeChangeHandlerAndCheckExistence(controllerChangeHandler)
+    activeControllerId = zkClient.getControllerId.getOrElse(-1)
+    if (wasActiveBeforeChange && !isActive) {
+      onControllerResignation()
+    }
   }
 
   private def elect(): Unit = {
-    val timestamp = time.milliseconds
     activeControllerId = zkClient.getControllerId.getOrElse(-1)
     /*
      * We can get here during the initial startup and the handleDeleted ZK callback. Because of the potential race condition,
@@ -1223,22 +1209,27 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     }
 
     try {
-      zkClient.registerController(config.brokerId, timestamp)
-      info(s"${config.brokerId} successfully elected as the controller")
+      val (epoch, epochZkVersion) = zkClient.registerControllerAndIncrementControllerEpoch(config.brokerId)
+      controllerContext.epoch = epoch
+      controllerContext.epochZkVersion = epochZkVersion
       activeControllerId = config.brokerId
+
+      info(s"${config.brokerId} successfully elected as the controller. Epoch incremented to ${controllerContext.epoch} " +
+        s"and epoch zk version is now ${controllerContext.epochZkVersion}")
+
       onControllerFailover()
     } catch {
-      case _: NodeExistsException =>
-        // If someone else has written the path, then
-        activeControllerId = zkClient.getControllerId.getOrElse(-1)
+      case e: ControllerMovedException =>
+        maybeResign()
 
         if (activeControllerId != -1)
-          debug(s"Broker $activeControllerId was elected as controller instead of broker ${config.brokerId}")
+          debug(s"Broker $activeControllerId was elected as controller instead of broker ${config.brokerId}", e)
         else
-          warn("A controller has been elected but just resigned, this will result in another round of election")
+          warn("A controller has been elected but just resigned, this will result in another round of election", e)
 
-      case e2: Throwable =>
-        error(s"Error while electing or becoming controller on broker ${config.brokerId}", e2)
+      case t: Throwable =>
+        error(s"Error while electing or becoming controller on broker ${config.brokerId}. " +
+          s"Trigger controller movement immediately", t)
         triggerControllerMove()
     }
   }
@@ -1321,7 +1312,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
         onBrokerLogDirFailure(brokerIds)
       } finally {
         // delete processed children
-        zkClient.deleteLogDirEventNotifications(sequenceNumbers)
+        zkClient.deleteLogDirEventNotifications(sequenceNumbers, controllerContext.epochZkVersion)
       }
     }
   }
@@ -1336,7 +1327,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
       val existingPartitionReplicaAssignment = newPartitionReplicaAssignment.filter(p =>
         existingPartitions.contains(p._1.partition.toString))
 
-      zkClient.setTopicAssignment(topic, existingPartitionReplicaAssignment)
+      zkClient.setTopicAssignment(topic, existingPartitionReplicaAssignment, controllerContext.epochZkVersion)
     }
 
     override def process(): Unit = {
@@ -1377,7 +1368,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
       val nonExistentTopics = topicsToBeDeleted -- controllerContext.allTopics
       if (nonExistentTopics.nonEmpty) {
         warn(s"Ignoring request to delete non-existing topics ${nonExistentTopics.mkString(",")}")
-        zkClient.deleteTopicDeletions(nonExistentTopics.toSeq)
+        zkClient.deleteTopicDeletions(nonExistentTopics.toSeq, controllerContext.epochZkVersion)
       }
       topicsToBeDeleted --= nonExistentTopics
       if (config.deleteTopicEnable) {
@@ -1396,7 +1387,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
       } else {
         // If delete topic is disabled remove entries under zookeeper path : /admin/delete_topics
         info(s"Removing $topicsToBeDeleted since delete topic is disabled")
-        zkClient.deleteTopicDeletions(topicsToBeDeleted.toSeq)
+        zkClient.deleteTopicDeletions(topicsToBeDeleted.toSeq, controllerContext.epochZkVersion)
       }
     }
   }
@@ -1469,7 +1460,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
         }
       } finally {
         // delete the notifications
-        zkClient.deleteIsrChangeNotifications(sequenceNumbers)
+        zkClient.deleteIsrChangeNotifications(sequenceNumbers, controllerContext.epochZkVersion)
       }
     }
 
@@ -1504,12 +1495,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     override def state = ControllerState.ControllerChange
 
     override def process(): Unit = {
-      val wasActiveBeforeChange = isActive
-      zkClient.registerZNodeChangeHandlerAndCheckExistence(controllerChangeHandler)
-      activeControllerId = zkClient.getControllerId.getOrElse(-1)
-      if (wasActiveBeforeChange && !isActive) {
-        onControllerResignation()
-      }
+      maybeResign()
     }
   }
 
@@ -1517,12 +1503,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti
     override def state = ControllerState.ControllerChange
 
     override def process(): Unit = {
-      val wasActiveBeforeChange = isActive
-      zkClient.registerZNodeChangeHandlerAndCheckExistence(controllerChangeHandler)
-      activeControllerId = zkClient.getControllerId.getOrElse(-1)
-      if (wasActiveBeforeChange && !isActive) {
-        onControllerResignation()
-      }
+      maybeResign()
       elect()
     }
   }
diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
index 3a0ac190fc1f0..663ee8da9b465 100755
--- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala
@@ -23,6 +23,7 @@ import kafka.utils.Logging
 import kafka.zk.{KafkaZkClient, TopicPartitionStateZNode}
 import kafka.zk.KafkaZkClient.UpdateLeaderAndIsrResult
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.errors.ControllerMovedException
 import org.apache.zookeeper.KeeperException
 import org.apache.zookeeper.KeeperException.Code
 
@@ -132,6 +133,9 @@ class PartitionStateMachine(config: KafkaConfig,
         doHandleStateChanges(partitions, targetState, partitionLeaderElectionStrategyOpt)
         controllerBrokerRequestBatch.sendRequestsToBrokers(controllerContext.epoch)
       } catch {
+        case e: ControllerMovedException =>
+          error(s"Controller moved to another broker when moving some partitions to $targetState state", e)
+          throw e
         case e: Throwable => error(s"Error while moving some partitions to $targetState state", e)
       }
     }
@@ -250,8 +254,11 @@ class PartitionStateMachine(config: KafkaConfig,
       partition -> leaderIsrAndControllerEpoch
     }.toMap
     val createResponses = try {
-      zkClient.createTopicPartitionStatesRaw(leaderIsrAndControllerEpochs)
+      zkClient.createTopicPartitionStatesRaw(leaderIsrAndControllerEpochs, controllerContext.epochZkVersion)
     } catch {
+      case e: ControllerMovedException =>
+        error("Controller moved to another broker when trying to create the topic partition state znode", e)
+        throw e
       case e: Exception =>
         partitionsWithLiveReplicas.foreach { case (partition,_) => logFailedStateChange(partition, partitionState(partition), NewPartition, e) }
         Seq.empty
@@ -361,7 +368,7 @@ class PartitionStateMachine(config: KafkaConfig,
     val recipientsPerPartition = partitionsWithLeaders.map { case (partition, _, recipients) => partition -> recipients }.toMap
     val adjustedLeaderAndIsrs = partitionsWithLeaders.map { case (partition, leaderAndIsrOpt, _) => partition -> leaderAndIsrOpt.get }.toMap
     val UpdateLeaderAndIsrResult(successfulUpdates, updatesToRetry, failedUpdates) = zkClient.updateLeaderAndIsr(
-      adjustedLeaderAndIsrs, controllerContext.epoch)
+      adjustedLeaderAndIsrs, controllerContext.epoch, controllerContext.epochZkVersion)
     successfulUpdates.foreach { case (partition, leaderAndIsr) =>
       val replicas = controllerContext.partitionReplicaAssignment(partition)
       val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(leaderAndIsr, controllerContext.epoch)
diff --git a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
index 1ab8a43dfde21..433ab5668379e 100644
--- a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
+++ b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala
@@ -23,6 +23,7 @@ import kafka.utils.Logging
 import kafka.zk.{KafkaZkClient, TopicPartitionStateZNode}
 import kafka.zk.KafkaZkClient.UpdateLeaderAndIsrResult
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.errors.ControllerMovedException
 import org.apache.zookeeper.KeeperException.Code
 
 import scala.collection.mutable
@@ -106,6 +107,9 @@ class ReplicaStateMachine(config: KafkaConfig,
         }
         controllerBrokerRequestBatch.sendRequestsToBrokers(controllerContext.epoch)
       } catch {
+        case e: ControllerMovedException =>
+          error(s"Controller moved to another broker when moving some replicas to $targetState state", e)
+          throw e
         case e: Throwable => error(s"Error while moving some replicas to $targetState state", e)
       }
     }
@@ -299,7 +303,7 @@ class ReplicaStateMachine(config: KafkaConfig,
       leaderAndIsr.newLeaderAndIsr(newLeader, adjustedIsr)
     }
     val UpdateLeaderAndIsrResult(successfulUpdates, updatesToRetry, failedUpdates) = zkClient.updateLeaderAndIsr(
-      adjustedLeaderAndIsrs, controllerContext.epoch)
+      adjustedLeaderAndIsrs, controllerContext.epoch, controllerContext.epochZkVersion)
     val exceptionsForPartitionsWithNoLeaderAndIsrInZk = partitionsWithNoLeaderAndIsrInZk.flatMap { partition =>
       if (!topicDeletionManager.isTopicQueuedUpForDeletion(partition.topic)) {
         val exception = new StateChangeFailedException(s"Failed to change state of replica $replicaId for partition $partition since the leader and isr path in zookeeper is empty")
diff --git a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
index 8d93ef2fa2da0..1ef79be794f89 100755
--- a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
+++ b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
@@ -92,7 +92,7 @@ class TopicDeletionManager(controller: KafkaController,
     } else {
       // if delete topic is disabled clean the topic entries under /admin/delete_topics
       info(s"Removing $initialTopicsToBeDeleted since delete topic is disabled")
-      zkClient.deleteTopicDeletions(initialTopicsToBeDeleted.toSeq)
+      zkClient.deleteTopicDeletions(initialTopicsToBeDeleted.toSeq, controllerContext.epochZkVersion)
     }
   }
 
@@ -251,9 +251,9 @@ class TopicDeletionManager(controller: KafkaController,
     controller.replicaStateMachine.handleStateChanges(replicasForDeletedTopic.toSeq, NonExistentReplica)
     topicsToBeDeleted -= topic
     topicsWithDeletionStarted -= topic
-    zkClient.deleteTopicZNode(topic)
-    zkClient.deleteTopicConfigs(Seq(topic))
-    zkClient.deleteTopicDeletions(Seq(topic))
+    zkClient.deleteTopicZNode(topic, controllerContext.epochZkVersion)
+    zkClient.deleteTopicConfigs(Seq(topic), controllerContext.epochZkVersion)
+    zkClient.deleteTopicDeletions(Seq(topic), controllerContext.epochZkVersion)
     controllerContext.removeTopic(topic)
   }
 
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 59581e738cf1f..2393daad003fa 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -166,7 +166,7 @@ class ReplicaManager(val config: KafkaConfig,
   }
 
   /* epoch of the controller that last changed the leader */
-  @volatile var controllerEpoch: Int = KafkaController.InitialControllerEpoch - 1
+  @volatile var controllerEpoch: Int = KafkaController.InitialControllerEpoch
   private val localBrokerId = config.brokerId
   private val allPartitions = new Pool[TopicPartition, Partition](valueFactory = Some(tp =>
     new Partition(tp.topic, tp.partition, time, this)))
diff --git a/core/src/main/scala/kafka/zk/KafkaZkClient.scala b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
index c8079654d6959..a12abb439638e 100644
--- a/core/src/main/scala/kafka/zk/KafkaZkClient.scala
+++ b/core/src/main/scala/kafka/zk/KafkaZkClient.scala
@@ -21,24 +21,27 @@ import java.util.Properties
 import com.yammer.metrics.core.MetricName
 import kafka.api.LeaderAndIsr
 import kafka.cluster.Broker
-import kafka.controller.LeaderIsrAndControllerEpoch
+import kafka.controller.{KafkaController, LeaderIsrAndControllerEpoch}
 import kafka.log.LogConfig
 import kafka.metrics.KafkaMetricsGroup
-import kafka.security.auth.SimpleAclAuthorizer.{VersionedAcls, NoAcls}
+import kafka.security.auth.SimpleAclAuthorizer.{NoAcls, VersionedAcls}
 import kafka.security.auth.{Acl, Resource, ResourceType}
 import kafka.server.ConfigType
 import kafka.utils.Logging
 import kafka.zookeeper._
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.resource.PatternType
+import org.apache.kafka.common.errors.ControllerMovedException
 import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation}
 import org.apache.kafka.common.utils.{Time, Utils}
-import org.apache.zookeeper.KeeperException.{Code, NodeExistsException}
+import org.apache.zookeeper.KeeperException.{BadVersionException, Code, ConnectionLossException, NodeExistsException}
+import org.apache.zookeeper.OpResult.{ErrorResult, SetDataResult}
 import org.apache.zookeeper.data.{ACL, Stat}
 import org.apache.zookeeper.{CreateMode, KeeperException, ZooKeeper}
 
 import scala.collection.mutable.ArrayBuffer
 import scala.collection.{Seq, mutable}
+import scala.collection.JavaConverters._
 
 /**
  * Provides higher level Kafka-specific operations on top of the pipelined [[kafka.zookeeper.ZooKeeperClient]].
@@ -86,14 +89,75 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   }
 
   /**
-   * Registers a given broker in zookeeper as the controller.
+   * Registers a given broker in zookeeper as the controller and increments controller epoch.
+   * @return the (updated controller epoch, epoch zkVersion) tuple
    * @param controllerId the id of the broker that is to be registered as the controller.
-   * @param timestamp the timestamp of the controller election.
-   * @throws KeeperException if an error is returned by ZooKeeper.
-   */
-  def registerController(controllerId: Int, timestamp: Long): Unit = {
-    val path = ControllerZNode.path
-    checkedEphemeralCreate(path, ControllerZNode.encode(controllerId, timestamp))
+   * @throws ControllerMovedException if fail to create /controller or fail to increment controller epoch.
+   */
+  def registerControllerAndIncrementControllerEpoch(controllerId: Int): (Int, Int) = {
+    val timestamp = time.milliseconds()
+
+    // Read /controller_epoch to get the current controller epoch and zkVersion,
+    // create /controller_epoch with initial value if not exists
+    val (curEpoch, curEpochZkVersion) = getControllerEpoch
+      .map(e => (e._1, e._2.getVersion))
+      .getOrElse(maybeCreateControllerEpochZNode())
+
+    // Create /controller and update /controller_epoch atomically
+    val newControllerEpoch = curEpoch + 1
+    val expectedControllerEpochZkVersion = curEpochZkVersion
+
+    debug(s"Try to create ${ControllerZNode.path} and increment controller epoch to $newControllerEpoch with expected controller epoch zkVersion $expectedControllerEpochZkVersion")
+
+    def checkControllerAndEpoch(): (Int, Int) = {
+      val curControllerId = getControllerId.getOrElse(throw new ControllerMovedException(
+        s"The ephemeral node at ${ControllerZNode.path} went away while checking whether the controller election succeeds. " +
+          s"Aborting controller startup procedure"))
+      if (controllerId == curControllerId) {
+        val (epoch, stat)  = getControllerEpoch.getOrElse(
+          throw new IllegalStateException(s"${ControllerEpochZNode.path} existed before but goes away while trying to read it"))
+
+        // If the epoch is the same as newControllerEpoch, it is safe to infer that the returned epoch zkVersion
+        // is associated with the current broker during controller election because we already knew that the zk
+        // transaction succeeds based on the controller znode verification. Other rounds of controller
+        // election will result in larger epoch number written in zk.
+        if (epoch == newControllerEpoch)
+          return (newControllerEpoch, stat.getVersion)
+      }
+      throw new ControllerMovedException("Controller moved to another broker. Aborting controller startup procedure")
+    }
+
+    def tryCreateControllerZNodeAndIncrementEpoch(): (Int, Int) = {
+      try {
+        val transaction = zooKeeperClient.createTransaction()
+        transaction.create(ControllerZNode.path, ControllerZNode.encode(controllerId, timestamp),
+          acls(ControllerZNode.path).asJava, CreateMode.EPHEMERAL)
+        transaction.setData(ControllerEpochZNode.path, ControllerEpochZNode.encode(newControllerEpoch), expectedControllerEpochZkVersion)
+        val results = transaction.commit()
+        val setDataResult = results.get(1).asInstanceOf[SetDataResult]
+        (newControllerEpoch, setDataResult.getStat.getVersion)
+      } catch {
+        case _: NodeExistsException | _: BadVersionException => checkControllerAndEpoch()
+        case _: ConnectionLossException =>
+          zooKeeperClient.waitUntilConnected()
+          tryCreateControllerZNodeAndIncrementEpoch()
+      }
+    }
+
+    tryCreateControllerZNodeAndIncrementEpoch()
+  }
+
+  private def maybeCreateControllerEpochZNode(): (Int, Int) = {
+    createControllerEpochRaw(KafkaController.InitialControllerEpoch).resultCode match {
+      case Code.OK =>
+        info(s"Successfully created ${ControllerEpochZNode.path} with initial epoch ${KafkaController.InitialControllerEpoch}")
+        (KafkaController.InitialControllerEpoch, KafkaController.InitialControllerEpochZkVersion)
+      case Code.NODEEXISTS =>
+        val (epoch, stat) = getControllerEpoch.getOrElse(throw new IllegalStateException(s"${ControllerEpochZNode.path} existed before but goes away while trying to read it"))
+        (epoch, stat.getVersion)
+      case code =>
+        throw KeeperException.create(code)
+    }
   }
 
   def updateBrokerInfo(brokerInfo: BrokerInfo): Unit = {
@@ -119,13 +183,15 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Sets topic partition states for the given partitions.
    * @param leaderIsrAndControllerEpochs the partition states of each partition whose state we wish to set.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @return sequence of SetDataResponse whose contexts are the partitions they are associated with.
    */
-  def setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs: Map[TopicPartition, LeaderIsrAndControllerEpoch]): Seq[SetDataResponse] = {
+  def setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs: Map[TopicPartition, LeaderIsrAndControllerEpoch], expectedControllerEpochZkVersion: Int): Seq[SetDataResponse] = {
     val setDataRequests = leaderIsrAndControllerEpochs.map { case (partition, leaderIsrAndControllerEpoch) =>
       val path = TopicPartitionStateZNode.path(partition)
       val data = TopicPartitionStateZNode.encode(leaderIsrAndControllerEpoch)
-      SetDataRequest(path, data, leaderIsrAndControllerEpoch.leaderAndIsr.zkVersion, Some(partition))
+      SetDataRequest(path, data, leaderIsrAndControllerEpoch.leaderAndIsr.zkVersion, Some(partition),
+        controllerZkVersionCheck(expectedControllerEpochZkVersion))
     }
     retryRequestsUntilConnected(setDataRequests.toSeq)
   }
@@ -133,15 +199,16 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Creates topic partition state znodes for the given partitions.
    * @param leaderIsrAndControllerEpochs the partition states of each partition whose state we wish to set.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @return sequence of CreateResponse whose contexts are the partitions they are associated with.
    */
-  def createTopicPartitionStatesRaw(leaderIsrAndControllerEpochs: Map[TopicPartition, LeaderIsrAndControllerEpoch]): Seq[CreateResponse] = {
-    createTopicPartitions(leaderIsrAndControllerEpochs.keys.map(_.topic).toSet.toSeq)
-    createTopicPartition(leaderIsrAndControllerEpochs.keys.toSeq)
+  def createTopicPartitionStatesRaw(leaderIsrAndControllerEpochs: Map[TopicPartition, LeaderIsrAndControllerEpoch], expectedControllerEpochZkVersion: Int): Seq[CreateResponse] = {
+    createTopicPartitions(leaderIsrAndControllerEpochs.keys.map(_.topic).toSet.toSeq, expectedControllerEpochZkVersion)
+    createTopicPartition(leaderIsrAndControllerEpochs.keys.toSeq, expectedControllerEpochZkVersion)
     val createRequests = leaderIsrAndControllerEpochs.map { case (partition, leaderIsrAndControllerEpoch) =>
       val path = TopicPartitionStateZNode.path(partition)
       val data = TopicPartitionStateZNode.encode(leaderIsrAndControllerEpoch)
-      CreateRequest(path, data, acls(path), CreateMode.PERSISTENT, Some(partition))
+      CreateRequest(path, data, acls(path), CreateMode.PERSISTENT, Some(partition), controllerZkVersionCheck(expectedControllerEpochZkVersion))
     }
     retryRequestsUntilConnected(createRequests.toSeq)
   }
@@ -172,9 +239,10 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
    * Update the partition states of multiple partitions in zookeeper.
    * @param leaderAndIsrs The partition states to update.
    * @param controllerEpoch The current controller epoch.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @return UpdateLeaderAndIsrResult instance containing per partition results.
    */
-  def updateLeaderAndIsr(leaderAndIsrs: Map[TopicPartition, LeaderAndIsr], controllerEpoch: Int): UpdateLeaderAndIsrResult = {
+  def updateLeaderAndIsr(leaderAndIsrs: Map[TopicPartition, LeaderAndIsr], controllerEpoch: Int, expectedControllerEpochZkVersion: Int): UpdateLeaderAndIsrResult = {
     val successfulUpdates = mutable.Map.empty[TopicPartition, LeaderAndIsr]
     val updatesToRetry = mutable.Buffer.empty[TopicPartition]
     val failed = mutable.Map.empty[TopicPartition, Exception]
@@ -182,8 +250,9 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
       partition -> LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)
     }
     val setDataResponses = try {
-      setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs)
+      setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs, expectedControllerEpochZkVersion)
     } catch {
+      case e: ControllerMovedException => throw e
       case e: Exception =>
         leaderAndIsrs.keys.foreach(partition => failed.put(partition, e))
         return UpdateLeaderAndIsrResult(successfulUpdates.toMap, updatesToRetry, failed.toMap)
@@ -381,10 +450,12 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
    * Sets the topic znode with the given assignment.
    * @param topic the topic whose assignment is being set.
    * @param assignment the partition to replica mapping to set for the given topic
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @return SetDataResponse
    */
-  def setTopicAssignmentRaw(topic: String, assignment: collection.Map[TopicPartition, Seq[Int]]): SetDataResponse = {
-    val setDataRequest = SetDataRequest(TopicZNode.path(topic), TopicZNode.encode(assignment), ZkVersion.MatchAnyVersion)
+  def setTopicAssignmentRaw(topic: String, assignment: collection.Map[TopicPartition, Seq[Int]], expectedControllerEpochZkVersion: Int): SetDataResponse = {
+    val setDataRequest = SetDataRequest(TopicZNode.path(topic), TopicZNode.encode(assignment), ZkVersion.MatchAnyVersion,
+      zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
     retryRequestUntilConnected(setDataRequest)
   }
 
@@ -392,10 +463,11 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
    * Sets the topic znode with the given assignment.
    * @param topic the topic whose assignment is being set.
    * @param assignment the partition to replica mapping to set for the given topic
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @throws KeeperException if there is an error while setting assignment
    */
-  def setTopicAssignment(topic: String, assignment: Map[TopicPartition, Seq[Int]]) = {
-    val setDataResponse = setTopicAssignmentRaw(topic, assignment)
+  def setTopicAssignment(topic: String, assignment: Map[TopicPartition, Seq[Int]], expectedControllerEpochZkVersion: Int = ZkVersion.MatchAnyVersion) = {
+    val setDataResponse = setTopicAssignmentRaw(topic, assignment, expectedControllerEpochZkVersion)
     setDataResponse.maybeThrow
   }
 
@@ -443,11 +515,12 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   /**
    * Deletes all log dir event notifications.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteLogDirEventNotifications(): Unit = {
+  def deleteLogDirEventNotifications(expectedControllerEpochZkVersion: Int): Unit = {
     val getChildrenResponse = retryRequestUntilConnected(GetChildrenRequest(LogDirEventNotificationZNode.path))
     if (getChildrenResponse.resultCode == Code.OK) {
-      deleteLogDirEventNotifications(getChildrenResponse.children.map(LogDirEventNotificationSequenceZNode.sequenceNumber))
+      deleteLogDirEventNotifications(getChildrenResponse.children.map(LogDirEventNotificationSequenceZNode.sequenceNumber), expectedControllerEpochZkVersion)
     } else if (getChildrenResponse.resultCode != Code.NONODE) {
       getChildrenResponse.maybeThrow
     }
@@ -456,10 +529,12 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Deletes the log dir event notifications associated with the given sequence numbers.
    * @param sequenceNumbers the sequence numbers associated with the log dir event notifications to be deleted.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteLogDirEventNotifications(sequenceNumbers: Seq[String]): Unit = {
+  def deleteLogDirEventNotifications(sequenceNumbers: Seq[String], expectedControllerEpochZkVersion: Int): Unit = {
     val deleteRequests = sequenceNumbers.map { sequenceNumber =>
-      DeleteRequest(LogDirEventNotificationSequenceZNode.path(sequenceNumber), ZkVersion.MatchAnyVersion)
+      DeleteRequest(LogDirEventNotificationSequenceZNode.path(sequenceNumber), ZkVersion.MatchAnyVersion,
+        zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
     }
     retryRequestsUntilConnected(deleteRequests)
   }
@@ -677,9 +752,11 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Remove the given topics from the topics marked for deletion.
    * @param topics the topics to remove.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteTopicDeletions(topics: Seq[String]): Unit = {
-    val deleteRequests = topics.map(topic => DeleteRequest(DeleteTopicsTopicZNode.path(topic), ZkVersion.MatchAnyVersion))
+  def deleteTopicDeletions(topics: Seq[String], expectedControllerEpochZkVersion: Int): Unit = {
+    val deleteRequests = topics.map(topic => DeleteRequest(DeleteTopicsTopicZNode.path(topic), ZkVersion.MatchAnyVersion,
+      zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion)))
     retryRequestsUntilConnected(deleteRequests)
   }
 
@@ -708,18 +785,20 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
    * exists or not.
    *
    * @param reassignment the reassignment to set on the reassignment znode
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @throws KeeperException if there is an error while setting or creating the znode
    */
-  def setOrCreatePartitionReassignment(reassignment: collection.Map[TopicPartition, Seq[Int]]): Unit = {
+  def setOrCreatePartitionReassignment(reassignment: collection.Map[TopicPartition, Seq[Int]], expectedControllerEpochZkVersion: Int): Unit = {
 
     def set(reassignmentData: Array[Byte]): SetDataResponse = {
-      val setDataRequest = SetDataRequest(ReassignPartitionsZNode.path, reassignmentData, ZkVersion.MatchAnyVersion)
+      val setDataRequest = SetDataRequest(ReassignPartitionsZNode.path, reassignmentData, ZkVersion.MatchAnyVersion,
+        zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
       retryRequestUntilConnected(setDataRequest)
     }
 
     def create(reassignmentData: Array[Byte]): CreateResponse = {
       val createRequest = CreateRequest(ReassignPartitionsZNode.path, reassignmentData, acls(ReassignPartitionsZNode.path),
-        CreateMode.PERSISTENT)
+        CreateMode.PERSISTENT, zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
       retryRequestUntilConnected(createRequest)
     }
 
@@ -744,9 +823,11 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   /**
    * Deletes the partition reassignment znode.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deletePartitionReassignment(): Unit = {
-    val deleteRequest = DeleteRequest(ReassignPartitionsZNode.path, ZkVersion.MatchAnyVersion)
+  def deletePartitionReassignment(expectedControllerEpochZkVersion: Int): Unit = {
+    val deleteRequest = DeleteRequest(ReassignPartitionsZNode.path, ZkVersion.MatchAnyVersion,
+      zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
     retryRequestUntilConnected(deleteRequest)
   }
 
@@ -851,11 +932,12 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   /**
    * Deletes all isr change notifications.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteIsrChangeNotifications(): Unit = {
+  def deleteIsrChangeNotifications(expectedControllerEpochZkVersion: Int): Unit = {
     val getChildrenResponse = retryRequestUntilConnected(GetChildrenRequest(IsrChangeNotificationZNode.path))
     if (getChildrenResponse.resultCode == Code.OK) {
-      deleteIsrChangeNotifications(getChildrenResponse.children.map(IsrChangeNotificationSequenceZNode.sequenceNumber))
+      deleteIsrChangeNotifications(getChildrenResponse.children.map(IsrChangeNotificationSequenceZNode.sequenceNumber), expectedControllerEpochZkVersion)
     } else if (getChildrenResponse.resultCode != Code.NONODE) {
       getChildrenResponse.maybeThrow
     }
@@ -864,10 +946,12 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Deletes the isr change notifications associated with the given sequence numbers.
    * @param sequenceNumbers the sequence numbers associated with the isr change notifications to be deleted.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteIsrChangeNotifications(sequenceNumbers: Seq[String]): Unit = {
+  def deleteIsrChangeNotifications(sequenceNumbers: Seq[String], expectedControllerEpochZkVersion: Int): Unit = {
     val deleteRequests = sequenceNumbers.map { sequenceNumber =>
-      DeleteRequest(IsrChangeNotificationSequenceZNode.path(sequenceNumber), ZkVersion.MatchAnyVersion)
+      DeleteRequest(IsrChangeNotificationSequenceZNode.path(sequenceNumber), ZkVersion.MatchAnyVersion,
+        zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
     }
     retryRequestsUntilConnected(deleteRequests)
   }
@@ -897,9 +981,11 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   /**
    * Deletes the preferred replica election znode.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deletePreferredReplicaElection(): Unit = {
-    val deleteRequest = DeleteRequest(PreferredReplicaElectionZNode.path, ZkVersion.MatchAnyVersion)
+  def deletePreferredReplicaElection(expectedControllerEpochZkVersion: Int): Unit = {
+    val deleteRequest = DeleteRequest(PreferredReplicaElectionZNode.path, ZkVersion.MatchAnyVersion,
+      zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
     retryRequestUntilConnected(deleteRequest)
   }
 
@@ -919,9 +1005,11 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   /**
    * Deletes the controller znode.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteController(): Unit = {
-    val deleteRequest = DeleteRequest(ControllerZNode.path, ZkVersion.MatchAnyVersion)
+  def deleteController(expectedControllerEpochZkVersion: Int): Unit = {
+    val deleteRequest = DeleteRequest(ControllerZNode.path, ZkVersion.MatchAnyVersion,
+      zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion))
     retryRequestUntilConnected(deleteRequest)
   }
 
@@ -944,17 +1032,20 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Recursively deletes the topic znode.
    * @param topic the topic whose topic znode we wish to delete.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteTopicZNode(topic: String): Unit = {
-    deleteRecursive(TopicZNode.path(topic))
+  def deleteTopicZNode(topic: String, expectedControllerEpochZkVersion: Int): Unit = {
+    deleteRecursive(TopicZNode.path(topic), expectedControllerEpochZkVersion)
   }
 
   /**
    * Deletes the topic configs for the given topics.
    * @param topics the topics whose configs we wish to delete.
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    */
-  def deleteTopicConfigs(topics: Seq[String]): Unit = {
-    val deleteRequests = topics.map(topic => DeleteRequest(ConfigEntityZNode.path(ConfigType.Topic, topic), ZkVersion.MatchAnyVersion))
+  def deleteTopicConfigs(topics: Seq[String], expectedControllerEpochZkVersion: Int): Unit = {
+    val deleteRequests = topics.map(topic => DeleteRequest(ConfigEntityZNode.path(ConfigType.Topic, topic),
+      ZkVersion.MatchAnyVersion, zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion)))
     retryRequestsUntilConnected(deleteRequests)
   }
 
@@ -1403,18 +1494,19 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
   /**
    * Deletes the given zk path recursively
    * @param path
+   * @param expectedControllerEpochZkVersion expected controller epoch zkVersion.
    * @return true if path gets deleted successfully, false if root path doesn't exist
    * @throws KeeperException if there is an error while deleting the znodes
    */
-  def deleteRecursive(path: String): Boolean = {
+  def deleteRecursive(path: String, expectedControllerEpochZkVersion: Int = ZkVersion.MatchAnyVersion): Boolean = {
     val getChildrenResponse = retryRequestUntilConnected(GetChildrenRequest(path))
     getChildrenResponse.resultCode match {
       case Code.OK =>
-        getChildrenResponse.children.foreach(child => deleteRecursive(s"$path/$child"))
-        val deleteResponse = retryRequestUntilConnected(DeleteRequest(path, ZkVersion.MatchAnyVersion))
-        if (deleteResponse.resultCode != Code.OK && deleteResponse.resultCode != Code.NONODE) {
+        getChildrenResponse.children.foreach(child => deleteRecursive(s"$path/$child", expectedControllerEpochZkVersion))
+        val deleteResponse = retryRequestUntilConnected(DeleteRequest(path, ZkVersion.MatchAnyVersion,
+          zkVersionCheck = controllerZkVersionCheck(expectedControllerEpochZkVersion)))
+        if (deleteResponse.resultCode != Code.OK && deleteResponse.resultCode != Code.NONODE)
           throw deleteResponse.resultException.get
-        }
         true
       case Code.NONODE => false
       case _ => throw getChildrenResponse.resultException.get
@@ -1468,18 +1560,18 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
 
   }
 
-  private def createTopicPartition(partitions: Seq[TopicPartition]): Seq[CreateResponse] = {
+  private def createTopicPartition(partitions: Seq[TopicPartition], expectedControllerEpochZkVersion: Int): Seq[CreateResponse] = {
     val createRequests = partitions.map { partition =>
       val path = TopicPartitionZNode.path(partition)
-      CreateRequest(path, null, acls(path), CreateMode.PERSISTENT, Some(partition))
+      CreateRequest(path, null, acls(path), CreateMode.PERSISTENT, Some(partition), controllerZkVersionCheck(expectedControllerEpochZkVersion))
     }
     retryRequestsUntilConnected(createRequests)
   }
 
-  private def createTopicPartitions(topics: Seq[String]): Seq[CreateResponse] = {
+  private def createTopicPartitions(topics: Seq[String], expectedControllerEpochZkVersion: Int):Seq[CreateResponse] = {
     val createRequests = topics.map { topic =>
       val path = TopicPartitionsZNode.path(topic)
-      CreateRequest(path, null, acls(path), CreateMode.PERSISTENT, Some(topic))
+      CreateRequest(path, null, acls(path), CreateMode.PERSISTENT, Some(topic), controllerZkVersionCheck(expectedControllerEpochZkVersion))
     }
     retryRequestsUntilConnected(createRequests)
   }
@@ -1513,13 +1605,16 @@ class KafkaZkClient private (zooKeeperClient: ZooKeeperClient, isSecure: Boolean
         requestResponsePairs.foreach { case (request, response) =>
           if (response.resultCode == Code.CONNECTIONLOSS)
             remainingRequests += request
-          else
+          else {
+            maybeThrowControllerMoveException(response)
             responses += response
+          }
         }
 
         if (remainingRequests.nonEmpty)
           zooKeeperClient.waitUntilConnected()
       } else {
+        batchResponses.foreach(maybeThrowControllerMoveException)
         remainingRequests.clear()
         responses ++= batchResponses
       }
@@ -1599,4 +1694,31 @@ object KafkaZkClient {
       time, metricGroup, metricType)
     new KafkaZkClient(zooKeeperClient, isSecure, time)
   }
+
+
+  private def controllerZkVersionCheck(version: Int): Option[ZkVersionCheck] = {
+    if (version < KafkaController.InitialControllerEpochZkVersion)
+      None
+    else
+      Some(ZkVersionCheck(ControllerEpochZNode.path, version))
+  }
+
+  private def maybeThrowControllerMoveException(response: AsyncResponse): Unit = {
+    response.zkVersionCheckResult match {
+      case Some(zkVersionCheckResult) =>
+        val zkVersionCheck = zkVersionCheckResult.zkVersionCheck
+        if (zkVersionCheck.checkPath.equals(ControllerEpochZNode.path))
+          zkVersionCheckResult.opResult match {
+            case errorResult: ErrorResult =>
+              val errorCode = Code.get(errorResult.getErr)
+              if (errorCode == Code.BADVERSION)
+              // Throw ControllerMovedException when the zkVersionCheck is performed on the controller epoch znode and the check fails
+                throw new ControllerMovedException(s"Controller epoch zkVersion check fails. Expected zkVersion = ${zkVersionCheck.expectedZkVersion}")
+              else if (errorCode != Code.OK)
+                throw KeeperException.create(errorCode, zkVersionCheck.checkPath)
+            case _ =>
+          }
+      case None =>
+    }
+  }
 }
diff --git a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
index 97ec9a44c36f3..59304143715b3 100755
--- a/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
+++ b/core/src/main/scala/kafka/zookeeper/ZooKeeperClient.scala
@@ -17,21 +17,23 @@
 
 package kafka.zookeeper
 
+import java.util
 import java.util.Locale
 import java.util.concurrent.locks.{ReentrantLock, ReentrantReadWriteLock}
-import java.util.concurrent.{ArrayBlockingQueue, ConcurrentHashMap, CountDownLatch, Semaphore, TimeUnit}
+import java.util.concurrent._
 
 import com.yammer.metrics.core.{Gauge, MetricName}
 import kafka.metrics.KafkaMetricsGroup
 import kafka.utils.CoreUtils.{inLock, inReadLock, inWriteLock}
 import kafka.utils.{KafkaScheduler, Logging}
 import org.apache.kafka.common.utils.Time
-import org.apache.zookeeper.AsyncCallback.{ACLCallback, Children2Callback, DataCallback, StatCallback, StringCallback, VoidCallback}
+import org.apache.zookeeper.AsyncCallback._
 import org.apache.zookeeper.KeeperException.Code
+import org.apache.zookeeper.OpResult.{CreateResult, SetDataResult}
 import org.apache.zookeeper.Watcher.Event.{EventType, KeeperState}
 import org.apache.zookeeper.ZooKeeper.States
 import org.apache.zookeeper.data.{ACL, Stat}
-import org.apache.zookeeper.{CreateMode, KeeperException, WatchedEvent, Watcher, ZooKeeper}
+import org.apache.zookeeper._
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.Set
@@ -156,6 +158,10 @@ class ZooKeeperClient(connectString: String,
       responseQueue.asScala.toBuffer
     }
   }
+  
+  def createTransaction(): Transaction = {
+    zooKeeper.transaction()
+  }
 
   // Visibility to override for testing
   private[zookeeper] def send[Req <: AsyncRequest](request: Req)(processResponse: Req#Response => Unit): Unit = {
@@ -166,44 +172,76 @@ class ZooKeeperClient(connectString: String,
 
     val sendTimeMs = time.hiResClockMs()
     request match {
-      case ExistsRequest(path, ctx) =>
+      case ExistsRequest(path, ctx, _) =>
         zooKeeper.exists(path, shouldWatch(request), new StatCallback {
           override def processResult(rc: Int, path: String, ctx: Any, stat: Stat): Unit =
             callback(ExistsResponse(Code.get(rc), path, Option(ctx), stat, responseMetadata(sendTimeMs)))
         }, ctx.orNull)
-      case GetDataRequest(path, ctx) =>
+      case GetDataRequest(path, ctx, _) =>
         zooKeeper.getData(path, shouldWatch(request), new DataCallback {
           override def processResult(rc: Int, path: String, ctx: Any, data: Array[Byte], stat: Stat): Unit =
             callback(GetDataResponse(Code.get(rc), path, Option(ctx), data, stat, responseMetadata(sendTimeMs)))
         }, ctx.orNull)
-      case GetChildrenRequest(path, ctx) =>
+      case GetChildrenRequest(path, ctx, _) =>
         zooKeeper.getChildren(path, shouldWatch(request), new Children2Callback {
           override def processResult(rc: Int, path: String, ctx: Any, children: java.util.List[String], stat: Stat): Unit =
             callback(GetChildrenResponse(Code.get(rc), path, Option(ctx),
               Option(children).map(_.asScala).getOrElse(Seq.empty), stat, responseMetadata(sendTimeMs)))
         }, ctx.orNull)
-      case CreateRequest(path, data, acl, createMode, ctx) =>
-        zooKeeper.create(path, data, acl.asJava, createMode, new StringCallback {
-          override def processResult(rc: Int, path: String, ctx: Any, name: String): Unit =
-            callback(CreateResponse(Code.get(rc), path, Option(ctx), name, responseMetadata(sendTimeMs)))
-        }, ctx.orNull)
-      case SetDataRequest(path, data, version, ctx) =>
-        zooKeeper.setData(path, data, version, new StatCallback {
-          override def processResult(rc: Int, path: String, ctx: Any, stat: Stat): Unit =
-            callback(SetDataResponse(Code.get(rc), path, Option(ctx), stat, responseMetadata(sendTimeMs)))
-        }, ctx.orNull)
-      case DeleteRequest(path, version, ctx) =>
-        zooKeeper.delete(path, version, new VoidCallback {
-          override def processResult(rc: Int, path: String, ctx: Any): Unit =
-            callback(DeleteResponse(Code.get(rc), path, Option(ctx), responseMetadata(sendTimeMs)))
-        }, ctx.orNull)
-      case GetAclRequest(path, ctx) =>
+      case CreateRequest(path, data, acl, createMode, ctx, zkVersionCheck) =>
+        if (zkVersionCheck.isEmpty)
+          zooKeeper.create(path, data, acl.asJava, createMode, new StringCallback {
+            override def processResult(rc: Int, path: String, ctx: Any, name: String): Unit =
+              callback(CreateResponse(Code.get(rc), path, Option(ctx), name, responseMetadata(sendTimeMs)))
+          }, ctx.orNull)
+        else
+          zooKeeper.multi(Seq(zkVersionCheck.get.checkOp, Op.create(path, data, acl.asJava, createMode)).asJava, new MultiCallback {
+            override def processResult(rc: Int, multiOpPath: String, ctx: scala.Any, opResults: util.List[OpResult]): Unit = {
+              val (zkVersionCheckOpResult, requestOpResult) = (opResults.get(0), opResults.get(1))
+              val name = requestOpResult match {
+                case c: CreateResult => c.getPath
+                case _ => null
+              }
+              callback(CreateResponse(Code.get(rc), path, Option(ctx), name, responseMetadata(sendTimeMs), 
+                Some(ZkVersionCheckResult(zkVersionCheck.get, zkVersionCheckOpResult))))
+            }}, ctx.orNull)
+      case SetDataRequest(path, data, version, ctx, zkVersionCheck) =>
+        if (zkVersionCheck.isEmpty)
+          zooKeeper.setData(path, data, version, new StatCallback {
+            override def processResult(rc: Int, path: String, ctx: Any, stat: Stat): Unit =
+              callback(SetDataResponse(Code.get(rc), path, Option(ctx), stat, responseMetadata(sendTimeMs)))
+          }, ctx.orNull)
+        else
+          zooKeeper.multi(Seq(zkVersionCheck.get.checkOp, Op.setData(path, data, version)).asJava, new MultiCallback {
+            override def processResult(rc: Int, multiOpPath: String, ctx: scala.Any, opResults: util.List[OpResult]): Unit = {
+              val (zkVersionCheckOpResult, requestOpResult) = (opResults.get(0), opResults.get(1))
+              val stat = requestOpResult match {
+                case s: SetDataResult => s.getStat
+                case _ => null
+              }
+              callback(SetDataResponse(Code.get(rc), path, Option(ctx), stat, responseMetadata(sendTimeMs), 
+                Some(ZkVersionCheckResult(zkVersionCheck.get, zkVersionCheckOpResult))))
+            }}, ctx.orNull)
+      case DeleteRequest(path, version, ctx, zkVersionCheck) =>
+        if (zkVersionCheck.isEmpty)
+          zooKeeper.delete(path, version, new VoidCallback {
+            override def processResult(rc: Int, path: String, ctx: Any): Unit =
+              callback(DeleteResponse(Code.get(rc), path, Option(ctx), responseMetadata(sendTimeMs)))
+          }, ctx.orNull)
+        else
+          zooKeeper.multi(Seq(zkVersionCheck.get.checkOp, Op.delete(path, version)).asJava, new MultiCallback {
+            override def processResult(rc: Int, multiOpPath: String, ctx: scala.Any, opResults: util.List[OpResult]): Unit = {
+              val (zkVersionCheckOpResult, _) = (opResults.get(0), opResults.get(1))
+              callback(DeleteResponse(Code.get(rc), path, Option(ctx), responseMetadata(sendTimeMs), 
+                Some(ZkVersionCheckResult(zkVersionCheck.get, zkVersionCheckOpResult))))
+            }}, ctx.orNull)
+      case GetAclRequest(path, ctx, _) =>
         zooKeeper.getACL(path, null, new ACLCallback {
           override def processResult(rc: Int, path: String, ctx: Any, acl: java.util.List[ACL], stat: Stat): Unit = {
             callback(GetAclResponse(Code.get(rc), path, Option(ctx), Option(acl).map(_.asScala).getOrElse(Seq.empty),
               stat, responseMetadata(sendTimeMs)))
-        }}, ctx.orNull)
-      case SetAclRequest(path, acl, version, ctx) =>
+          }}, ctx.orNull)
+      case SetAclRequest(path, acl, version, ctx, _) =>
         zooKeeper.setACL(path, acl.asJava, version, new StatCallback {
           override def processResult(rc: Int, path: String, ctx: Any, stat: Stat): Unit =
             callback(SetAclResponse(Code.get(rc), path, Option(ctx), stat, responseMetadata(sendTimeMs)))
@@ -329,7 +367,7 @@ class ZooKeeperClient(connectString: String,
   private[kafka] def currentZooKeeper: ZooKeeper = inReadLock(initializationLock) {
     zooKeeper
   }
-  
+
   private def reinitialize(): Unit = {
     // Initialization callbacks are invoked outside of the lock to avoid deadlock potential since their completion
     // may require additional Zookeeper requests, which will block to acquire the initialization lock
@@ -447,45 +485,54 @@ sealed trait AsyncRequest {
   type Response <: AsyncResponse
   def path: String
   def ctx: Option[Any]
+  def zkVersionCheck: Option[ZkVersionCheck]
+}
+
+case class ZkVersionCheck(checkPath: String, expectedZkVersion: Int) {
+  def checkOp: Op = Op.check(checkPath, expectedZkVersion)
 }
 
+case class ZkVersionCheckResult(zkVersionCheck: ZkVersionCheck, opResult: OpResult)
+
 case class CreateRequest(path: String, data: Array[Byte], acl: Seq[ACL], createMode: CreateMode,
-                         ctx: Option[Any] = None) extends AsyncRequest {
+                         ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = CreateResponse
 }
 
-case class DeleteRequest(path: String, version: Int, ctx: Option[Any] = None) extends AsyncRequest {
+case class DeleteRequest(path: String, version: Int, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = DeleteResponse
 }
 
-case class ExistsRequest(path: String, ctx: Option[Any] = None) extends AsyncRequest {
+case class ExistsRequest(path: String, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = ExistsResponse
 }
 
-case class GetDataRequest(path: String, ctx: Option[Any] = None) extends AsyncRequest {
+case class GetDataRequest(path: String, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = GetDataResponse
 }
 
-case class SetDataRequest(path: String, data: Array[Byte], version: Int, ctx: Option[Any] = None) extends AsyncRequest {
+case class SetDataRequest(path: String, data: Array[Byte], version: Int, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = SetDataResponse
 }
 
-case class GetAclRequest(path: String, ctx: Option[Any] = None) extends AsyncRequest {
+case class GetAclRequest(path: String, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = GetAclResponse
 }
 
-case class SetAclRequest(path: String, acl: Seq[ACL], version: Int, ctx: Option[Any] = None) extends AsyncRequest {
+case class SetAclRequest(path: String, acl: Seq[ACL], version: Int, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = SetAclResponse
 }
 
-case class GetChildrenRequest(path: String, ctx: Option[Any] = None) extends AsyncRequest {
+case class GetChildrenRequest(path: String, ctx: Option[Any] = None, zkVersionCheck: Option[ZkVersionCheck] = None) extends AsyncRequest {
   type Response = GetChildrenResponse
 }
 
+
 sealed abstract class AsyncResponse {
   def resultCode: Code
   def path: String
   def ctx: Option[Any]
+  def zkVersionCheckResult: Option[ZkVersionCheckResult]
 
   /** Return None if the result code is OK and KeeperException otherwise. */
   def resultException: Option[KeeperException] =
@@ -506,17 +553,22 @@ case class ResponseMetadata(sendTimeMs: Long, receivedTimeMs: Long) {
   def responseTimeMs: Long = receivedTimeMs - sendTimeMs
 }
 
-case class CreateResponse(resultCode: Code, path: String, ctx: Option[Any], name: String, metadata: ResponseMetadata) extends AsyncResponse
-case class DeleteResponse(resultCode: Code, path: String, ctx: Option[Any], metadata: ResponseMetadata) extends AsyncResponse
-case class ExistsResponse(resultCode: Code, path: String, ctx: Option[Any], stat: Stat, metadata: ResponseMetadata) extends AsyncResponse
+case class CreateResponse(resultCode: Code, path: String, ctx: Option[Any], name: String,
+                          metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
+case class DeleteResponse(resultCode: Code, path: String, ctx: Option[Any],
+                          metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
+case class ExistsResponse(resultCode: Code, path: String, ctx: Option[Any], stat: Stat,
+                          metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
 case class GetDataResponse(resultCode: Code, path: String, ctx: Option[Any], data: Array[Byte], stat: Stat,
-                           metadata: ResponseMetadata) extends AsyncResponse
-case class SetDataResponse(resultCode: Code, path: String, ctx: Option[Any], stat: Stat, metadata: ResponseMetadata) extends AsyncResponse
+                           metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
+case class SetDataResponse(resultCode: Code, path: String, ctx: Option[Any], stat: Stat,
+                           metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
 case class GetAclResponse(resultCode: Code, path: String, ctx: Option[Any], acl: Seq[ACL], stat: Stat,
-                          metadata: ResponseMetadata) extends AsyncResponse
-case class SetAclResponse(resultCode: Code, path: String, ctx: Option[Any], stat: Stat, metadata: ResponseMetadata) extends AsyncResponse
+                          metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
+case class SetAclResponse(resultCode: Code, path: String, ctx: Option[Any], stat: Stat,
+                          metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
 case class GetChildrenResponse(resultCode: Code, path: String, ctx: Option[Any], children: Seq[String], stat: Stat,
-                               metadata: ResponseMetadata) extends AsyncResponse
+                               metadata: ResponseMetadata, zkVersionCheckResult: Option[ZkVersionCheckResult] = None) extends AsyncResponse
 
 class ZooKeeperClientException(message: String) extends RuntimeException(message)
 class ZooKeeperClientExpiredException(message: String) extends ZooKeeperClientException(message)
diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
index 4f40b27f01942..b44c239cfbd48 100644
--- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsClusterTest.scala
@@ -20,7 +20,7 @@ import kafka.common.AdminCommandFailedException
 import kafka.server.{KafkaConfig, KafkaServer}
 import kafka.utils.TestUtils._
 import kafka.utils.{Logging, TestUtils}
-import kafka.zk.{ReassignPartitionsZNode, ZooKeeperTestHarness}
+import kafka.zk.{ReassignPartitionsZNode, ZkVersion, ZooKeeperTestHarness}
 import org.junit.Assert.{assertEquals, assertTrue}
 import org.junit.{After, Before, Test}
 import kafka.admin.ReplicationQuotaUtils._
@@ -613,7 +613,7 @@ class ReassignPartitionsClusterTest extends ZooKeeperTestHarness with Logging {
     )
 
     // Set znode directly to avoid non-existent topic validation
-    zkClient.setOrCreatePartitionReassignment(firstMove)
+    zkClient.setOrCreatePartitionReassignment(firstMove, ZkVersion.MatchAnyVersion)
 
     servers.foreach(_.startup())
     waitForReassignmentToComplete()
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala
index e5753e58030ec..e0a753cacaf74 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerEventManagerTest.scala
@@ -54,7 +54,7 @@ class ControllerEventManagerTest {
     val controllerStats = new ControllerStats
     val eventProcessedListenerCount = new AtomicInteger
     controllerEventManager = new ControllerEventManager(0, controllerStats.rateAndTimeMetrics,
-      _ => eventProcessedListenerCount.incrementAndGet)
+      _ => eventProcessedListenerCount.incrementAndGet, () => ())
     controllerEventManager.start()
 
     val initialTimerCount = timer(metricName).count
diff --git a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
index 5e5d84f9626d2..dc4076a27c75e 100644
--- a/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ControllerIntegrationTest.scala
@@ -17,23 +17,29 @@
 
 package kafka.controller
 
-import java.util.concurrent.LinkedBlockingQueue
+import java.util.Properties
+import java.util.concurrent.{CountDownLatch, LinkedBlockingQueue}
 
 import com.yammer.metrics.Metrics
 import com.yammer.metrics.core.Timer
 import kafka.api.LeaderAndIsr
 import kafka.server.{KafkaConfig, KafkaServer}
 import kafka.utils.TestUtils
-import kafka.zk.{PreferredReplicaElectionZNode, ZooKeeperTestHarness}
+import kafka.zk._
 import org.junit.{After, Before, Test}
 import org.junit.Assert.{assertEquals, assertTrue}
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.errors.ControllerMovedException
+import org.apache.log4j.Level
+import kafka.utils.LogCaptureAppender
 
 import scala.collection.JavaConverters._
 import scala.util.Try
 
 class ControllerIntegrationTest extends ZooKeeperTestHarness {
   var servers = Seq.empty[KafkaServer]
+  val firstControllerEpoch = KafkaController.InitialControllerEpoch + 1
+  val firstControllerEpochZkVersion = KafkaController.InitialControllerEpochZkVersion + 1
 
   @Before
   override def setUp() {
@@ -51,30 +57,30 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
   def testEmptyCluster(): Unit = {
     servers = makeServers(1)
     TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
-    waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "broker failed to set controller epoch")
+    waitUntilControllerEpoch(firstControllerEpoch, "broker failed to set controller epoch")
   }
 
   @Test
   def testControllerEpochPersistsWhenAllBrokersDown(): Unit = {
     servers = makeServers(1)
     TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
-    waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "broker failed to set controller epoch")
+    waitUntilControllerEpoch(firstControllerEpoch, "broker failed to set controller epoch")
     servers.head.shutdown()
     servers.head.awaitShutdown()
     TestUtils.waitUntilTrue(() => !zkClient.getControllerId.isDefined, "failed to kill controller")
-    waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "controller epoch was not persisted after broker failure")
+    waitUntilControllerEpoch(firstControllerEpoch, "controller epoch was not persisted after broker failure")
   }
 
   @Test
   def testControllerMoveIncrementsControllerEpoch(): Unit = {
     servers = makeServers(1)
     TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
-    waitUntilControllerEpoch(KafkaController.InitialControllerEpoch, "broker failed to set controller epoch")
+    waitUntilControllerEpoch(firstControllerEpoch, "broker failed to set controller epoch")
     servers.head.shutdown()
     servers.head.awaitShutdown()
     servers.head.startup()
     TestUtils.waitUntilTrue(() => zkClient.getControllerId.isDefined, "failed to elect a controller")
-    waitUntilControllerEpoch(KafkaController.InitialControllerEpoch + 1, "controller epoch was not incremented after controller move")
+    waitUntilControllerEpoch(firstControllerEpoch + 1, "controller epoch was not incremented after controller move")
   }
 
   @Test
@@ -83,7 +89,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(0))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
+    waitForPartitionState(tp, firstControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic creation")
   }
 
@@ -97,7 +103,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId, controllerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers.take(1))
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch,
+    waitForPartitionState(tp, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic creation")
   }
 
@@ -109,8 +115,8 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val assignment = Map(tp0.partition -> Seq(0))
     val expandedAssignment = Map(tp0 -> Seq(0), tp1 -> Seq(0))
     TestUtils.createTopic(zkClient, tp0.topic, partitionReplicaAssignment = assignment, servers = servers)
-    zkClient.setTopicAssignment(tp0.topic, expandedAssignment)
-    waitForPartitionState(tp1, KafkaController.InitialControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
+    zkClient.setTopicAssignment(tp0.topic, expandedAssignment, firstControllerEpochZkVersion)
+    waitForPartitionState(tp1, firstControllerEpoch, 0, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic partition expansion")
     TestUtils.waitUntilMetadataIsPropagated(servers, tp1.topic, tp1.partition)
   }
@@ -127,8 +133,8 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     TestUtils.createTopic(zkClient, tp0.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkClient.setTopicAssignment(tp0.topic, expandedAssignment)
-    waitForPartitionState(tp1, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch,
+    zkClient.setTopicAssignment(tp0.topic, expandedAssignment, firstControllerEpochZkVersion)
+    waitForPartitionState(tp1, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic partition expansion")
     TestUtils.waitUntilMetadataIsPropagated(Seq(servers(controllerId)), tp1.topic, tp1.partition)
   }
@@ -147,7 +153,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val reassignment = Map(tp -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     zkClient.createPartitionReassignment(reassignment)
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 3,
+    waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 3,
       "failed to get expected partition state after partition reassignment")
     TestUtils.waitUntilTrue(() =>  zkClient.getReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
       "failed to get updated partition assignment on topic znode after partition reassignment")
@@ -169,8 +175,9 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    zkClient.setOrCreatePartitionReassignment(reassignment)
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
+    val controller = getController()
+    zkClient.setOrCreatePartitionReassignment(reassignment, controller.kafkaController.controllerContext.epochZkVersion)
+    waitForPartitionState(tp, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state during partition reassignment with offline replica")
     TestUtils.waitUntilTrue(() => zkClient.reassignPartitionsInProgress(),
       "partition reassignment path should remain while reassignment in progress")
@@ -188,10 +195,10 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
     zkClient.createPartitionReassignment(reassignment)
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
+    waitForPartitionState(tp, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state during partition reassignment with offline replica")
     servers(otherBrokerId).startup()
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 4,
+    waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 4,
       "failed to get expected partition state after partition reassignment")
     TestUtils.waitUntilTrue(() => zkClient.getReplicaAssignmentForTopics(Set(tp.topic)) == reassignment,
       "failed to get updated partition assignment on topic znode after partition reassignment")
@@ -235,7 +242,7 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     zkClient.createPreferredReplicaElection(Set(tp))
     TestUtils.waitUntilTrue(() => !zkClient.pathExists(PreferredReplicaElectionZNode.path),
       "failed to remove preferred replica leader election path after giving up")
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
+    waitForPartitionState(tp, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state upon broker shutdown")
   }
 
@@ -249,10 +256,10 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
+    waitForPartitionState(tp, firstControllerEpoch, controllerId, LeaderAndIsr.initialLeaderEpoch + 1,
       "failed to get expected partition state upon broker shutdown")
     servers(otherBrokerId).startup()
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 2,
+    waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch + 2,
       "failed to get expected partition state upon broker startup")
   }
 
@@ -264,14 +271,14 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch,
+    waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic creation")
     servers(otherBrokerId).shutdown()
     servers(otherBrokerId).awaitShutdown()
     TestUtils.waitUntilTrue(() => {
       val leaderIsrAndControllerEpochMap = zkClient.getTopicPartitionStates(Seq(tp))
       leaderIsrAndControllerEpochMap.contains(tp) &&
-        isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), KafkaController.InitialControllerEpoch, LeaderAndIsr.NoLeader, LeaderAndIsr.initialLeaderEpoch + 1) &&
+        isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), firstControllerEpoch, LeaderAndIsr.NoLeader, LeaderAndIsr.initialLeaderEpoch + 1) &&
         leaderIsrAndControllerEpochMap(tp).leaderAndIsr.isr == List(otherBrokerId)
     }, "failed to get expected partition state after entire isr went offline")
   }
@@ -284,14 +291,14 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     val tp = new TopicPartition("t", 0)
     val assignment = Map(tp.partition -> Seq(otherBrokerId))
     TestUtils.createTopic(zkClient, tp.topic, partitionReplicaAssignment = assignment, servers = servers)
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch,
+    waitForPartitionState(tp, firstControllerEpoch, otherBrokerId, LeaderAndIsr.initialLeaderEpoch,
       "failed to get expected partition state upon topic creation")
     servers(1).shutdown()
     servers(1).awaitShutdown()
     TestUtils.waitUntilTrue(() => {
       val leaderIsrAndControllerEpochMap = zkClient.getTopicPartitionStates(Seq(tp))
       leaderIsrAndControllerEpochMap.contains(tp) &&
-        isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), KafkaController.InitialControllerEpoch, LeaderAndIsr.NoLeader, LeaderAndIsr.initialLeaderEpoch + 1) &&
+        isExpectedPartitionState(leaderIsrAndControllerEpochMap(tp), firstControllerEpoch, LeaderAndIsr.NoLeader, LeaderAndIsr.initialLeaderEpoch + 1) &&
         leaderIsrAndControllerEpochMap(tp).leaderAndIsr.isr == List(otherBrokerId)
     }, "failed to get expected partition state after entire isr went offline")
   }
@@ -341,18 +348,105 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
     assertTrue(servers.forall(_.apis.metadataCache.getPartitionInfo(topic,partition).get.basePartitionState.leader == 0))
   }
 
+  @Test
+  def testControllerMoveOnTopicCreation(): Unit = {
+    servers = makeServers(1)
+    TestUtils.waitUntilControllerElected(zkClient)
+    val tp = new TopicPartition("t", 0)
+    val assignment = Map(tp.partition -> Seq(0))
+
+    testControllerMove(() => {
+      val adminZkClient = new AdminZkClient(zkClient)
+      adminZkClient.createOrUpdateTopicPartitionAssignmentPathInZK(tp.topic, assignment, new Properties())
+    })
+  }
+
+  @Test
+  def testControllerMoveOnTopicDeletion(): Unit = {
+    servers = makeServers(1)
+    TestUtils.waitUntilControllerElected(zkClient)
+    val tp = new TopicPartition("t", 0)
+    val assignment = Map(tp.partition -> Seq(0))
+    TestUtils.createTopic(zkClient, tp.topic(), assignment, servers)
+
+    testControllerMove(() => {
+      val adminZkClient = new AdminZkClient(zkClient)
+      adminZkClient.deleteTopic(tp.topic())
+    })
+  }
+
+  @Test
+  def testControllerMoveOnPreferredReplicaElection(): Unit = {
+    servers = makeServers(1)
+    val tp = new TopicPartition("t", 0)
+    val assignment = Map(tp.partition -> Seq(0))
+    TestUtils.createTopic(zkClient, tp.topic(), assignment, servers)
+
+    testControllerMove(() => zkClient.createPreferredReplicaElection(Set(tp)))
+  }
+
+  @Test
+  def testControllerMoveOnPartitionReassignment(): Unit = {
+    servers = makeServers(1)
+    TestUtils.waitUntilControllerElected(zkClient)
+    val tp = new TopicPartition("t", 0)
+    val assignment = Map(tp.partition -> Seq(0))
+    TestUtils.createTopic(zkClient, tp.topic(), assignment, servers)
+
+    val reassignment = Map(tp -> Seq(0))
+    testControllerMove(() => zkClient.createPartitionReassignment(reassignment))
+  }
+
+  private def testControllerMove(fun: () => Unit): Unit = {
+    val controller = getController().kafkaController
+    val appender = LogCaptureAppender.createAndRegister()
+    val previousLevel = LogCaptureAppender.setClassLoggerLevel(controller.eventManager.thread.getClass, Level.INFO)
+
+    try {
+      TestUtils.waitUntilTrue(() => {
+        controller.eventManager.state == ControllerState.Idle
+      }, "Controller event thread is still busy")
+
+      val latch = new CountDownLatch(1)
+
+      // Let the controller event thread await on a latch before the pre-defined logic is triggered.
+      // This is used to make sure that when the event thread resumes and starts processing events, the controller has already moved.
+      controller.eventManager.put(KafkaController.AwaitOnLatch(latch))
+      // Execute pre-defined logic. This can be topic creation/deletion, preferred leader election, etc.
+      fun()
+
+      // Delete the controller path, re-create /controller znode to emulate controller movement
+      zkClient.deleteController(controller.controllerContext.epochZkVersion)
+      zkClient.registerControllerAndIncrementControllerEpoch(servers.size)
+
+      // Resume the controller event thread. At this point, the controller should see mismatch controller epoch zkVersion and resign
+      latch.countDown()
+      TestUtils.waitUntilTrue(() => !controller.isActive, "Controller fails to resign")
+
+      // Expect to capture the ControllerMovedException in the log of ControllerEventThread
+      val event = appender.getMessages.find(e => e.getLevel == Level.INFO
+        && e.getThrowableInformation != null
+        && e.getThrowableInformation.getThrowable.getClass.getName.equals(new ControllerMovedException("").getClass.getName))
+      assertTrue(event.isDefined)
+
+    } finally {
+      LogCaptureAppender.unregister(appender)
+      LogCaptureAppender.setClassLoggerLevel(controller.eventManager.thread.getClass, previousLevel)
+    }
+  }
+
   private def preferredReplicaLeaderElection(controllerId: Int, otherBroker: KafkaServer, tp: TopicPartition,
                                              replicas: Set[Int], leaderEpoch: Int): Unit = {
     otherBroker.shutdown()
     otherBroker.awaitShutdown()
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, controllerId, leaderEpoch + 1,
+    waitForPartitionState(tp, firstControllerEpoch, controllerId, leaderEpoch + 1,
       "failed to get expected partition state upon broker shutdown")
     otherBroker.startup()
     TestUtils.waitUntilTrue(() => zkClient.getInSyncReplicasForPartition(new TopicPartition(tp.topic, tp.partition)).get.toSet == replicas, "restarted broker failed to join in-sync replicas")
     zkClient.createPreferredReplicaElection(Set(tp))
     TestUtils.waitUntilTrue(() => !zkClient.pathExists(PreferredReplicaElectionZNode.path),
       "failed to remove preferred replica leader election path after completion")
-    waitForPartitionState(tp, KafkaController.InitialControllerEpoch, otherBroker.config.brokerId, leaderEpoch + 2,
+    waitForPartitionState(tp, firstControllerEpoch, otherBroker.config.brokerId, leaderEpoch + 2,
       "failed to get expected partition state upon broker startup")
   }
 
@@ -395,4 +489,9 @@ class ControllerIntegrationTest extends ZooKeeperTestHarness {
       .getOrElse(fail(s"Unable to find metric $metricName")).asInstanceOf[Timer]
   }
 
+  private def getController(): KafkaServer = {
+    val controllerId = TestUtils.waitUntilControllerElected(zkClient)
+    servers.filter(s => s.config.brokerId == controllerId).head
+  }
+
 }
diff --git a/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala b/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
index 6a587f3bd34b5..b89632e0a83d5 100644
--- a/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/PartitionStateMachineTest.scala
@@ -22,7 +22,7 @@ import kafka.server.KafkaConfig
 import kafka.utils.TestUtils
 import kafka.zk.{KafkaZkClient, TopicPartitionStateZNode}
 import kafka.zk.KafkaZkClient.UpdateLeaderAndIsrResult
-import kafka.zookeeper.{CreateResponse, GetDataResponse, ResponseMetadata, ZooKeeperClientException}
+import kafka.zookeeper._
 import org.apache.kafka.common.TopicPartition
 import org.apache.zookeeper.KeeperException.Code
 import org.apache.zookeeper.data.Stat
@@ -85,7 +85,7 @@ class PartitionStateMachineTest extends JUnitSuite {
     partitionState.put(partition, NewPartition)
     val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(LeaderAndIsr(brokerId, List(brokerId)), controllerEpoch)
     EasyMock.expect(mockControllerBrokerRequestBatch.newBatch())
-    EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch)))
+    EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch), controllerContext.epochZkVersion))
       .andReturn(Seq(CreateResponse(Code.OK, null, Some(partition), null, ResponseMetadata(0, 0))))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
       partition, leaderIsrAndControllerEpoch, Seq(brokerId), isNew = true))
@@ -103,7 +103,7 @@ class PartitionStateMachineTest extends JUnitSuite {
     partitionState.put(partition, NewPartition)
     val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(LeaderAndIsr(brokerId, List(brokerId)), controllerEpoch)
     EasyMock.expect(mockControllerBrokerRequestBatch.newBatch())
-    EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch)))
+    EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch), controllerContext.epochZkVersion))
       .andThrow(new ZooKeeperClientException("test"))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
@@ -119,7 +119,7 @@ class PartitionStateMachineTest extends JUnitSuite {
     partitionState.put(partition, NewPartition)
     val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(LeaderAndIsr(brokerId, List(brokerId)), controllerEpoch)
     EasyMock.expect(mockControllerBrokerRequestBatch.newBatch())
-    EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch)))
+    EasyMock.expect(mockZkClient.createTopicPartitionStatesRaw(Map(partition -> leaderIsrAndControllerEpoch), controllerContext.epochZkVersion))
       .andReturn(Seq(CreateResponse(Code.NODEEXISTS, null, Some(partition), null, ResponseMetadata(0, 0))))
     EasyMock.expect(mockControllerBrokerRequestBatch.sendRequestsToBrokers(controllerEpoch))
     EasyMock.replay(mockZkClient, mockControllerBrokerRequestBatch)
@@ -159,7 +159,7 @@ class PartitionStateMachineTest extends JUnitSuite {
 
     val leaderAndIsrAfterElection = leaderAndIsr.newLeader(brokerId)
     val updatedLeaderAndIsr = leaderAndIsrAfterElection.withZkVersion(2)
-    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch))
+    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> updatedLeaderAndIsr), Seq.empty, Map.empty))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
       partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), Seq(brokerId), isNew = false))
@@ -190,7 +190,7 @@ class PartitionStateMachineTest extends JUnitSuite {
 
     val leaderAndIsrAfterElection = leaderAndIsr.newLeaderAndIsr(otherBrokerId, List(otherBrokerId))
     val updatedLeaderAndIsr = leaderAndIsrAfterElection.withZkVersion(2)
-    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch))
+    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> updatedLeaderAndIsr), Seq.empty, Map.empty))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(otherBrokerId),
       partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), Seq(brokerId, otherBrokerId),
@@ -243,7 +243,7 @@ class PartitionStateMachineTest extends JUnitSuite {
       .andReturn((Map(partition.topic -> LogConfig()), Map.empty))
     val leaderAndIsrAfterElection = leaderAndIsr.newLeader(brokerId)
     val updatedLeaderAndIsr = leaderAndIsrAfterElection.withZkVersion(2)
-    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch))
+    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> leaderAndIsrAfterElection), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> updatedLeaderAndIsr), Seq.empty, Map.empty))
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(brokerId),
       partition, LeaderIsrAndControllerEpoch(updatedLeaderAndIsr, controllerEpoch), Seq(brokerId), isNew = false))
@@ -336,7 +336,7 @@ class PartitionStateMachineTest extends JUnitSuite {
       val updatedLeaderAndIsr = partitions.map { partition =>
         partition -> leaderAndIsr.newLeaderAndIsr(brokerId, List(brokerId))
       }.toMap
-      EasyMock.expect(mockZkClient.updateLeaderAndIsr(updatedLeaderAndIsr, controllerEpoch))
+      EasyMock.expect(mockZkClient.updateLeaderAndIsr(updatedLeaderAndIsr, controllerEpoch, controllerContext.epochZkVersion))
         .andReturn(UpdateLeaderAndIsrResult(updatedLeaderAndIsr, Seq.empty, Map.empty))
     }
     prepareMockToUpdateLeaderAndIsr()
diff --git a/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala b/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala
index c573c9f041e71..ef274fa4fa787 100644
--- a/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala
+++ b/core/src/test/scala/unit/kafka/controller/ReplicaStateMachineTest.scala
@@ -183,7 +183,7 @@ class ReplicaStateMachineTest extends JUnitSuite {
     EasyMock.expect(mockZkClient.getTopicPartitionStatesRaw(partitions)).andReturn(
       Seq(GetDataResponse(Code.OK, null, Some(partition),
         TopicPartitionStateZNode.encode(leaderIsrAndControllerEpoch), stat, ResponseMetadata(0, 0))))
-    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> adjustedLeaderAndIsr), controllerEpoch))
+    EasyMock.expect(mockZkClient.updateLeaderAndIsr(Map(partition -> adjustedLeaderAndIsr), controllerEpoch, controllerContext.epochZkVersion))
       .andReturn(UpdateLeaderAndIsrResult(Map(partition -> updatedLeaderAndIsr), Seq.empty, Map.empty))
     EasyMock.expect(mockTopicDeletionManager.isTopicQueuedUpForDeletion(partition.topic)).andReturn(false)
     EasyMock.expect(mockControllerBrokerRequestBatch.addLeaderAndIsrRequestForBrokers(Seq(otherBrokerId),
diff --git a/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala b/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala
new file mode 100644
index 0000000000000..80472e9b19f52
--- /dev/null
+++ b/core/src/test/scala/unit/kafka/utils/LogCaptureAppender.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.utils
+
+import org.apache.log4j.{AppenderSkeleton, Level, Logger}
+import org.apache.log4j.spi.LoggingEvent
+
+import scala.collection.mutable.ListBuffer
+
+class LogCaptureAppender extends AppenderSkeleton {
+  private val events: ListBuffer[LoggingEvent] = ListBuffer.empty
+
+  override protected def append(event: LoggingEvent): Unit = {
+    events.synchronized {
+      events += event
+    }
+  }
+
+  def getMessages: ListBuffer[LoggingEvent] = {
+    events.synchronized {
+      return events.clone()
+    }
+  }
+
+  override def close(): Unit =  {
+    events.synchronized {
+      events.clear()
+    }
+  }
+
+  override def requiresLayout: Boolean = false
+}
+
+object LogCaptureAppender {
+  def createAndRegister(): LogCaptureAppender = {
+    val logCaptureAppender: LogCaptureAppender = new LogCaptureAppender
+    Logger.getRootLogger.addAppender(logCaptureAppender)
+    logCaptureAppender
+  }
+
+  def setClassLoggerLevel(clazz: Class[_], logLevel: Level): Level = {
+    val logger = Logger.getLogger(clazz)
+    val previousLevel = logger.getLevel
+    Logger.getLogger(clazz).setLevel(logLevel)
+    previousLevel
+  }
+
+  def unregister(logCaptureAppender: LogCaptureAppender): Unit = {
+    Logger.getRootLogger.removeAppender(logCaptureAppender)
+  }
+}
diff --git a/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala b/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
index 3389161e78b3e..65273eb01a583 100644
--- a/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
+++ b/core/src/test/scala/unit/kafka/utils/ReplicationUtilsTest.scala
@@ -20,7 +20,7 @@ package kafka.utils
 import kafka.server.{KafkaConfig, ReplicaFetcherManager}
 import kafka.api.LeaderAndIsr
 import kafka.controller.LeaderIsrAndControllerEpoch
-import kafka.zk.{IsrChangeNotificationZNode, TopicZNode, ZooKeeperTestHarness}
+import kafka.zk._
 import org.apache.kafka.common.TopicPartition
 import org.junit.Assert._
 import org.junit.{Before, Test}
@@ -42,7 +42,7 @@ class ReplicationUtilsTest extends ZooKeeperTestHarness {
     val topicPartition = new TopicPartition(topic, partition)
     val leaderAndIsr = LeaderAndIsr(leader, leaderEpoch, isr, 1)
     val leaderIsrAndControllerEpoch = LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)
-    zkClient.createTopicPartitionStatesRaw(Map(topicPartition -> leaderIsrAndControllerEpoch))
+    zkClient.createTopicPartitionStatesRaw(Map(topicPartition -> leaderIsrAndControllerEpoch), ZkVersion.MatchAnyVersion)
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 48d7d3fd199ff..d42d02c210d17 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -36,7 +36,7 @@ import kafka.server._
 import kafka.server.checkpoints.OffsetCheckpointFile
 import Implicits._
 import kafka.controller.LeaderIsrAndControllerEpoch
-import kafka.zk.{AdminZkClient, BrokerIdsZNode, BrokerInfo, KafkaZkClient}
+import kafka.zk._
 import org.apache.kafka.clients.CommonClientConfigs
 import org.apache.kafka.clients.admin.{AdminClient, AlterConfigsResult, Config, ConfigEntry}
 import org.apache.kafka.clients.consumer._
@@ -635,7 +635,7 @@ object TestUtils extends Logging {
         .getOrElse(LeaderAndIsr(leader, List(leader)))
       topicPartition -> LeaderIsrAndControllerEpoch(newLeaderAndIsr, controllerEpoch)
     }
-    zkClient.setTopicPartitionStatesRaw(newLeaderIsrAndControllerEpochs)
+    zkClient.setTopicPartitionStatesRaw(newLeaderIsrAndControllerEpochs, ZkVersion.MatchAnyVersion)
   }
 
   /**
diff --git a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
index 9cffb517c0999..61ca3bbb6f841 100644
--- a/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
+++ b/core/src/test/scala/unit/kafka/zk/KafkaZkClientTest.scala
@@ -42,6 +42,7 @@ import scala.util.Random
 import kafka.controller.LeaderIsrAndControllerEpoch
 import kafka.zk.KafkaZkClient.UpdateLeaderAndIsrResult
 import kafka.zookeeper._
+import org.apache.kafka.common.errors.ControllerMovedException
 import org.apache.kafka.common.security.JaasUtils
 import org.apache.zookeeper.data.Stat
 
@@ -55,12 +56,14 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   val topicPartition11 = new TopicPartition(topic1, 1)
   val topicPartition20 = new TopicPartition(topic2, 0)
   val topicPartitions10_11 = Seq(topicPartition10, topicPartition11)
+  val controllerEpochZkVersion = 0
 
   var otherZkClient: KafkaZkClient = _
 
   @Before
   override def setUp(): Unit = {
     super.setUp()
+    zkClient.createControllerEpochRaw(1)
     otherZkClient = KafkaZkClient(zkConnect, zkAclsEnabled.getOrElse(JaasUtils.isZkSecurityEnabled), zkSessionTimeout,
       zkConnectionTimeout, zkMaxInFlightRequests, Time.SYSTEM)
   }
@@ -69,6 +72,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   override def tearDown(): Unit = {
     if (otherZkClient != null)
       otherZkClient.close()
+    zkClient.deletePath(ControllerEpochZNode.path)
     super.tearDown()
   }
 
@@ -99,15 +103,28 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     zkClient.createRecursive("/delete/some/random/path")
     assertTrue(zkClient.pathExists("/delete/some/random/path"))
-    zkClient.deleteRecursive("/delete")
-    assertFalse(zkClient.pathExists("/delete/some/random/path"))
-    assertFalse(zkClient.pathExists("/delete/some/random"))
-    assertFalse(zkClient.pathExists("/delete/some"))
+    assertTrue(zkClient.deleteRecursive("/delete"))
     assertFalse(zkClient.pathExists("/delete"))
 
     intercept[IllegalArgumentException](zkClient.deleteRecursive("delete-invalid-path"))
   }
 
+  @Test
+  def testDeleteRecursiveWithControllerEpochVersionCheck(): Unit = {
+    assertFalse(zkClient.deleteRecursive("/delete/does-not-exist", controllerEpochZkVersion))
+
+    zkClient.createRecursive("/delete/some/random/path")
+    assertTrue(zkClient.pathExists("/delete/some/random/path"))
+    intercept[ControllerMovedException](
+      zkClient.deleteRecursive("/delete", controllerEpochZkVersion + 1))
+
+    assertTrue(zkClient.deleteRecursive("/delete", controllerEpochZkVersion))
+    assertFalse(zkClient.pathExists("/delete"))
+
+    intercept[IllegalArgumentException](zkClient.deleteRecursive(
+      "delete-invalid-path", controllerEpochZkVersion))
+  }
+
   @Test
   def testCreateRecursive() {
     zkClient.createRecursive("/create-newrootpath")
@@ -268,7 +285,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   @Test
   def testIsrChangeNotificationsDeletion(): Unit = {
     // Should not fail even if parent node does not exist
-    zkClient.deleteIsrChangeNotifications(Seq("0000000000"))
+    zkClient.deleteIsrChangeNotifications(Seq("0000000000"), controllerEpochZkVersion)
 
     zkClient.createRecursive("/isr_change_notification")
 
@@ -276,13 +293,18 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     zkClient.propagateIsrChanges(Set(topicPartition10))
     zkClient.propagateIsrChanges(Set(topicPartition11))
 
-    zkClient.deleteIsrChangeNotifications(Seq("0000000001"))
+    // Should throw exception if the controllerEpochZkVersion does not match
+    intercept[ControllerMovedException](zkClient.deleteIsrChangeNotifications(Seq("0000000001"), controllerEpochZkVersion + 1))
+    // Delete should not succeed
+    assertEquals(Set("0000000000", "0000000001", "0000000002"), zkClient.getAllIsrChangeNotifications.toSet)
+
+    zkClient.deleteIsrChangeNotifications(Seq("0000000001"), controllerEpochZkVersion)
     // Should not fail if called on a non-existent notification
-    zkClient.deleteIsrChangeNotifications(Seq("0000000001"))
+    zkClient.deleteIsrChangeNotifications(Seq("0000000001"), controllerEpochZkVersion)
 
     assertEquals(Set("0000000000", "0000000002"), zkClient.getAllIsrChangeNotifications.toSet)
-    zkClient.deleteIsrChangeNotifications()
-    assertEquals(Seq.empty,zkClient.getAllIsrChangeNotifications)
+    zkClient.deleteIsrChangeNotifications(controllerEpochZkVersion)
+    assertEquals(Seq.empty, zkClient.getAllIsrChangeNotifications)
   }
 
   @Test
@@ -335,7 +357,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   @Test
   def testLogDirEventNotificationsDeletion(): Unit = {
     // Should not fail even if parent node does not exist
-    zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"))
+    zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"), controllerEpochZkVersion)
 
     zkClient.createRecursive("/log_dir_event_notification")
 
@@ -346,13 +368,16 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     zkClient.propagateLogDirEvent(brokerId)
     zkClient.propagateLogDirEvent(anotherBrokerId)
 
-    zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"))
+    intercept[ControllerMovedException](zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"), controllerEpochZkVersion + 1))
+    assertEquals(Seq("0000000000", "0000000001", "0000000002"), zkClient.getAllLogDirEventNotifications)
+
+    zkClient.deleteLogDirEventNotifications(Seq("0000000000", "0000000002"), controllerEpochZkVersion)
 
     assertEquals(Seq("0000000001"), zkClient.getAllLogDirEventNotifications)
 
     zkClient.propagateLogDirEvent(anotherBrokerId)
 
-    zkClient.deleteLogDirEventNotifications()
+    zkClient.deleteLogDirEventNotifications(controllerEpochZkVersion)
     assertEquals(Seq.empty, zkClient.getAllLogDirEventNotifications)
   }
 
@@ -368,14 +393,18 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
       new TopicPartition("topic_b", 0) -> Seq(4, 5),
       new TopicPartition("topic_c", 0) -> Seq(5, 3)
     )
-    zkClient.setOrCreatePartitionReassignment(reassignment)
+
+    // Should throw ControllerMovedException if the controller epoch zkVersion does not match
+    intercept[ControllerMovedException](zkClient.setOrCreatePartitionReassignment(reassignment, controllerEpochZkVersion + 1))
+
+    zkClient.setOrCreatePartitionReassignment(reassignment, controllerEpochZkVersion)
     assertEquals(reassignment, zkClient.getPartitionReassignment)
 
-    val updatedReassingment = reassignment - new TopicPartition("topic_b", 0)
-    zkClient.setOrCreatePartitionReassignment(updatedReassingment)
-    assertEquals(updatedReassingment, zkClient.getPartitionReassignment)
+    val updatedReassignment = reassignment - new TopicPartition("topic_b", 0)
+    zkClient.setOrCreatePartitionReassignment(updatedReassignment, controllerEpochZkVersion)
+    assertEquals(updatedReassignment, zkClient.getPartitionReassignment)
 
-    zkClient.deletePartitionReassignment()
+    zkClient.deletePartitionReassignment(controllerEpochZkVersion)
     assertEquals(Map.empty, zkClient.getPartitionReassignment)
 
     zkClient.createPartitionReassignment(reassignment)
@@ -513,9 +542,9 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
   @Test
   def testDeleteTopicZNode(): Unit = {
-    zkClient.deleteTopicZNode(topic1)
+    zkClient.deleteTopicZNode(topic1, controllerEpochZkVersion)
     zkClient.createRecursive(TopicZNode.path(topic1))
-    zkClient.deleteTopicZNode(topic1)
+    zkClient.deleteTopicZNode(topic1, controllerEpochZkVersion)
     assertFalse(zkClient.pathExists(TopicZNode.path(topic1)))
   }
 
@@ -530,7 +559,10 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     assertTrue(zkClient.isTopicMarkedForDeletion(topic1))
     assertEquals(Set(topic1, topic2), zkClient.getTopicDeletions.toSet)
 
-    zkClient.deleteTopicDeletions(Seq(topic1, topic2))
+    intercept[ControllerMovedException](zkClient.deleteTopicDeletions(Seq(topic1, topic2), controllerEpochZkVersion + 1))
+    assertEquals(Set(topic1, topic2), zkClient.getTopicDeletions.toSet)
+
+    zkClient.deleteTopicDeletions(Seq(topic1, topic2), controllerEpochZkVersion)
     assertTrue(zkClient.getTopicDeletions.isEmpty)
   }
 
@@ -564,7 +596,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     zkClient.setOrCreateEntityConfigs(ConfigType.Topic, topic2, logProps)
     assertEquals(Set(topic1, topic2), zkClient.getAllEntitiesWithConfig(ConfigType.Topic).toSet)
 
-    zkClient.deleteTopicConfigs(Seq(topic1, topic2))
+    zkClient.deleteTopicConfigs(Seq(topic1, topic2), controllerEpochZkVersion)
     assertTrue(zkClient.getEntityConfigs(ConfigType.Topic, topic1).isEmpty)
   }
 
@@ -742,22 +774,26 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
       Map(
         topicPartition10 -> (classOf[NoNodeException], "KeeperErrorCode = NoNode for /brokers/topics/topic1/partitions/0/state"),
         topicPartition11 -> (classOf[NoNodeException], "KeeperErrorCode = NoNode for /brokers/topics/topic1/partitions/1/state")),
-      zkClient.updateLeaderAndIsr(initialLeaderIsrs, controllerEpoch = 4))
+      zkClient.updateLeaderAndIsr(initialLeaderIsrs, controllerEpoch = 4, controllerEpochZkVersion))
+
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion)
 
-    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+    // Mismatch controller epoch zkVersion
+    intercept[ControllerMovedException](zkClient.updateLeaderAndIsr(initialLeaderIsrs, controllerEpoch = 4, controllerEpochZkVersion + 1))
 
+    // successful updates
     checkUpdateLeaderAndIsrResult(
       leaderIsrs(state = 1, zkVersion = 1),
       mutable.ArrayBuffer.empty,
       Map.empty,
-      zkClient.updateLeaderAndIsr(leaderIsrs(state = 1, zkVersion = 0),controllerEpoch = 4))
+      zkClient.updateLeaderAndIsr(leaderIsrs(state = 1, zkVersion = 0),controllerEpoch = 4, controllerEpochZkVersion))
 
     // Try to update with wrong ZK version
     checkUpdateLeaderAndIsrResult(
       Map.empty,
       ArrayBuffer(topicPartition10, topicPartition11),
       Map.empty,
-      zkClient.updateLeaderAndIsr(leaderIsrs(state = 1, zkVersion = 0),controllerEpoch = 4))
+      zkClient.updateLeaderAndIsr(leaderIsrs(state = 1, zkVersion = 0),controllerEpoch = 4, controllerEpochZkVersion))
 
     // Trigger successful, to be retried and failed partitions in same call
     val mixedState = Map(
@@ -770,7 +806,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
       ArrayBuffer(topicPartition11),
       Map(
         topicPartition20 -> (classOf[NoNodeException], "KeeperErrorCode = NoNode for /brokers/topics/topic2/partitions/0/state")),
-      zkClient.updateLeaderAndIsr(mixedState, controllerEpoch = 4))
+      zkClient.updateLeaderAndIsr(mixedState, controllerEpoch = 4, controllerEpochZkVersion))
   }
 
   private def checkGetDataResponse(
@@ -786,9 +822,9 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
       TopicPartitionStateZNode.decode(response.data, statWithVersion(zkVersion)))
   }
 
-  private def eraseMetadata(response: CreateResponse): CreateResponse =
-    response.copy(metadata = ResponseMetadata(0, 0))
-
+  private def eraseUncheckedInfoInCreateResponse(response: CreateResponse): CreateResponse =
+    response.copy(metadata = ResponseMetadata(0, 0), zkVersionCheckResult = None)
+  
   @Test
   def testGetTopicsAndPartitions(): Unit = {
     assertTrue(zkClient.getAllTopicsInCluster.isEmpty)
@@ -800,7 +836,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     assertTrue(zkClient.getAllPartitions.isEmpty)
 
-    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion)
     assertEquals(Set(topicPartition10, topicPartition11), zkClient.getAllPartitions)
   }
 
@@ -808,14 +844,17 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
   def testCreateAndGetTopicPartitionStatesRaw(): Unit = {
     zkClient.createRecursive(TopicZNode.path(topic1))
 
+    // Mismatch controller epoch zkVersion
+    intercept[ControllerMovedException](zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion + 1))
+
     assertEquals(
       Seq(
         CreateResponse(Code.OK, TopicPartitionStateZNode.path(topicPartition10), Some(topicPartition10),
           TopicPartitionStateZNode.path(topicPartition10), ResponseMetadata(0, 0)),
         CreateResponse(Code.OK, TopicPartitionStateZNode.path(topicPartition11), Some(topicPartition11),
           TopicPartitionStateZNode.path(topicPartition11), ResponseMetadata(0, 0))),
-      zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
-        .map(eraseMetadata).toList)
+      zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion)
+        .map(eraseUncheckedInfoInCreateResponse).toList)
 
     val getResponses = zkClient.getTopicPartitionStatesRaw(topicPartitions10_11)
     assertEquals(2, getResponses.size)
@@ -824,11 +863,9 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     // Trying to create existing topicPartition states fails
     assertEquals(
       Seq(
-        CreateResponse(Code.NODEEXISTS, TopicPartitionStateZNode.path(topicPartition10), Some(topicPartition10),
-          null, ResponseMetadata(0, 0)),
-        CreateResponse(Code.NODEEXISTS, TopicPartitionStateZNode.path(topicPartition11), Some(topicPartition11),
-          null, ResponseMetadata(0, 0))),
-      zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs).map(eraseMetadata).toList)
+        CreateResponse(Code.NODEEXISTS, TopicPartitionStateZNode.path(topicPartition10), Some(topicPartition10), null, ResponseMetadata(0, 0)),
+        CreateResponse(Code.NODEEXISTS, TopicPartitionStateZNode.path(topicPartition11), Some(topicPartition11), null, ResponseMetadata(0, 0))),
+      zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion).map(eraseUncheckedInfoInCreateResponse).toList)
   }
 
   @Test
@@ -837,7 +874,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     def expectedSetDataResponses(topicPartitions: TopicPartition*)(resultCode: Code, stat: Stat) =
       topicPartitions.map { topicPartition =>
         SetDataResponse(resultCode, TopicPartitionStateZNode.path(topicPartition),
-          Some(topicPartition), stat, ResponseMetadata(0, 0))
+          Some(topicPartition), stat, ResponseMetadata(0, 0), None)
       }
 
     zkClient.createRecursive(TopicZNode.path(topic1))
@@ -845,16 +882,18 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     // Trying to set non-existing topicPartition's data results in NONODE responses
     assertEquals(
       expectedSetDataResponses(topicPartition10, topicPartition11)(Code.NONODE, null),
-      zkClient.setTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs).map {
-        _.copy(metadata = ResponseMetadata(0, 0))}.toList)
+      zkClient.setTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion).map {
+        _.copy(metadata = ResponseMetadata(0, 0), zkVersionCheckResult = None)}.toList)
 
-    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion)
 
     assertEquals(
       expectedSetDataResponses(topicPartition10, topicPartition11)(Code.OK, statWithVersion(1)),
-      zkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 1, zkVersion = 0)).map {
-        eraseMetadataAndStat}.toList)
+      zkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 1, zkVersion = 0), controllerEpochZkVersion).map {
+        eraseUncheckedInfoInSetDataResponse}.toList)
 
+    // Mismatch controller epoch zkVersion
+    intercept[ControllerMovedException](zkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 1, zkVersion = 0), controllerEpochZkVersion + 1))
 
     val getResponses = zkClient.getTopicPartitionStatesRaw(topicPartitions10_11)
     assertEquals(2, getResponses.size)
@@ -863,8 +902,8 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     // Other ZK client can also write the state of a partition
     assertEquals(
       expectedSetDataResponses(topicPartition10, topicPartition11)(Code.OK, statWithVersion(2)),
-      otherZkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 2, zkVersion = 1)).map {
-        eraseMetadataAndStat}.toList)
+      otherZkClient.setTopicPartitionStatesRaw(leaderIsrAndControllerEpochs(state = 2, zkVersion = 1), controllerEpochZkVersion).map {
+        eraseUncheckedInfoInSetDataResponse}.toList)
   }
 
   @Test
@@ -881,7 +920,7 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
     zkClient.createRecursive(TopicZNode.path(topic1))
 
-    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs)
+    zkClient.createTopicPartitionStatesRaw(initialLeaderIsrAndControllerEpochs, controllerEpochZkVersion)
     assertEquals(
       initialLeaderIsrAndControllerEpochs,
       zkClient.getTopicPartitionStates(Seq(topicPartition10, topicPartition11))
@@ -906,36 +945,38 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
 
   }
 
-  private def eraseMetadataAndStat(response: SetDataResponse): SetDataResponse = {
+  private def eraseUncheckedInfoInSetDataResponse(response: SetDataResponse): SetDataResponse = {
     val stat = if (response.stat != null) statWithVersion(response.stat.getVersion) else null
-    response.copy(metadata = ResponseMetadata(0, 0), stat = stat)
+    response.copy(metadata = ResponseMetadata(0, 0), stat = stat, zkVersionCheckResult = None)
   }
 
   @Test
   def testControllerEpochMethods(): Unit = {
+    zkClient.deletePath(ControllerEpochZNode.path)
+
     assertEquals(None, zkClient.getControllerEpoch)
 
     assertEquals("Setting non existing nodes should return NONODE results",
       SetDataResponse(Code.NONODE, ControllerEpochZNode.path, None, null, ResponseMetadata(0, 0)),
-      eraseMetadataAndStat(zkClient.setControllerEpochRaw(1, 0)))
+      eraseUncheckedInfoInSetDataResponse(zkClient.setControllerEpochRaw(1, 0)))
 
     assertEquals("Creating non existing nodes is OK",
       CreateResponse(Code.OK, ControllerEpochZNode.path, None, ControllerEpochZNode.path, ResponseMetadata(0, 0)),
-      eraseMetadata(zkClient.createControllerEpochRaw(0)))
+      eraseUncheckedInfoInCreateResponse(zkClient.createControllerEpochRaw(0)))
     assertEquals(0, zkClient.getControllerEpoch.get._1)
 
     assertEquals("Attemt to create existing nodes should return NODEEXISTS",
       CreateResponse(Code.NODEEXISTS, ControllerEpochZNode.path, None, null, ResponseMetadata(0, 0)),
-      eraseMetadata(zkClient.createControllerEpochRaw(0)))
+      eraseUncheckedInfoInCreateResponse(zkClient.createControllerEpochRaw(0)))
 
     assertEquals("Updating existing nodes is OK",
       SetDataResponse(Code.OK, ControllerEpochZNode.path, None, statWithVersion(1), ResponseMetadata(0, 0)),
-      eraseMetadataAndStat(zkClient.setControllerEpochRaw(1, 0)))
+      eraseUncheckedInfoInSetDataResponse(zkClient.setControllerEpochRaw(1, 0)))
     assertEquals(1, zkClient.getControllerEpoch.get._1)
 
     assertEquals("Updating with wrong ZK version returns BADVERSION",
       SetDataResponse(Code.BADVERSION, ControllerEpochZNode.path, None, null, ResponseMetadata(0, 0)),
-      eraseMetadataAndStat(zkClient.setControllerEpochRaw(1, 0)))
+      eraseUncheckedInfoInSetDataResponse(zkClient.setControllerEpochRaw(1, 0)))
   }
 
   @Test
@@ -943,9 +984,9 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
     // No controller
     assertEquals(None, zkClient.getControllerId)
     // Create controller
-    zkClient.registerController(controllerId = 1, timestamp = 123456)
+    val (_, newEpochZkVersion) = zkClient.registerControllerAndIncrementControllerEpoch(controllerId = 1)
     assertEquals(Some(1), zkClient.getControllerId)
-    zkClient.deleteController()
+    zkClient.deleteController(newEpochZkVersion)
     assertEquals(None, zkClient.getControllerId)
   }
 
@@ -1002,7 +1043,11 @@ class KafkaZkClientTest extends ZooKeeperTestHarness {
       zkClient.createPreferredReplicaElection(electionPartitions)
     }
 
-    zkClient.deletePreferredReplicaElection()
+    // Mismatch controller epoch zkVersion
+    intercept[ControllerMovedException](zkClient.deletePreferredReplicaElection(controllerEpochZkVersion + 1))
+    assertEquals(electionPartitions, zkClient.getPreferredReplicaElection)
+
+    zkClient.deletePreferredReplicaElection(controllerEpochZkVersion)
     assertTrue(zkClient.getPreferredReplicaElection.isEmpty)
   }
 

From f348f10ef87925081fdf9455ace6d2a86179b483 Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Sat, 8 Sep 2018 06:10:59 +0530
Subject: [PATCH 0783/1847] KAFKA-7117: Support AdminClient API in AclCommand
 (KIP-332) (#5463)

Reviewers: Colin Patrick McCabe , Jun Rao 
---
 .../main/scala/kafka/admin/AclCommand.scala   | 280 +++++++++++++-----
 .../scala/kafka/security/SecurityUtils.scala  |   5 +-
 .../unit/kafka/admin/AclCommandTest.scala     |  89 ++++--
 docs/security.html                            |  24 +-
 4 files changed, 307 insertions(+), 91 deletions(-)

diff --git a/core/src/main/scala/kafka/admin/AclCommand.scala b/core/src/main/scala/kafka/admin/AclCommand.scala
index 31e6c53dc1142..c2dda33d5ab2c 100644
--- a/core/src/main/scala/kafka/admin/AclCommand.scala
+++ b/core/src/main/scala/kafka/admin/AclCommand.scala
@@ -17,17 +17,22 @@
 
 package kafka.admin
 
+import java.util.Properties
+
 import joptsimple._
 import joptsimple.util.EnumConverter
 import kafka.security.auth._
 import kafka.server.KafkaConfig
 import kafka.utils._
+import org.apache.kafka.clients.admin.{AdminClientConfig, AdminClient => JAdminClient}
+import org.apache.kafka.common.acl._
 import org.apache.kafka.common.security.JaasUtils
 import org.apache.kafka.common.security.auth.KafkaPrincipal
-import org.apache.kafka.common.utils.Utils
-import org.apache.kafka.common.resource.{PatternType, ResourcePatternFilter, Resource => JResource, ResourceType => JResourceType}
+import org.apache.kafka.common.utils.{SecurityUtils, Utils}
+import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, Resource => JResource, ResourceType => JResourceType}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 object AclCommand extends Logging {
 
@@ -52,13 +57,21 @@ object AclCommand extends Logging {
 
     opts.checkArgs()
 
+    val aclCommandService = {
+      if (opts.options.has(opts.bootstrapServerOpt)) {
+        new AdminClientService(opts)
+      } else {
+        new AuthorizerService(opts)
+      }
+    }
+
     try {
       if (opts.options.has(opts.addOpt))
-        addAcl(opts)
+        aclCommandService.addAcls()
       else if (opts.options.has(opts.removeOpt))
-        removeAcl(opts)
+        aclCommandService.removeAcls()
       else if (opts.options.has(opts.listOpt))
-        listAcl(opts)
+        aclCommandService.listAcls()
     } catch {
       case e: Throwable =>
         println(s"Error while executing ACL command: ${e.getMessage}")
@@ -67,91 +80,202 @@ object AclCommand extends Logging {
     }
   }
 
-  def withAuthorizer(opts: AclCommandOptions)(f: Authorizer => Unit) {
-    val defaultProps = Map(KafkaConfig.ZkEnableSecureAclsProp -> JaasUtils.isZkSecurityEnabled)
-    val authorizerProperties =
-      if (opts.options.has(opts.authorizerPropertiesOpt)) {
-        val authorizerProperties = opts.options.valuesOf(opts.authorizerPropertiesOpt).asScala
-        defaultProps ++ CommandLineUtils.parseKeyValueArgs(authorizerProperties, acceptMissingValue = false).asScala
-      } else {
-        defaultProps
+  sealed trait AclCommandService {
+    def addAcls(): Unit
+    def removeAcls(): Unit
+    def listAcls(): Unit
+  }
+
+  class AdminClientService(val opts: AclCommandOptions) extends AclCommandService with Logging {
+
+    private def withAdminClient(opts: AclCommandOptions)(f: JAdminClient => Unit) {
+      val props = if (opts.options.has(opts.commandConfigOpt))
+        Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+      else
+        new Properties()
+      props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+      val adminClient = JAdminClient.create(props)
+
+      try {
+        f(adminClient)
+      } finally {
+        adminClient.close()
       }
+    }
 
-    val authorizerClass = opts.options.valueOf(opts.authorizerOpt)
-    val authZ = CoreUtils.createObject[Authorizer](authorizerClass)
-    try {
-      authZ.configure(authorizerProperties.asJava)
-      f(authZ)
+    def addAcls(): Unit = {
+      val resourceToAcl = getResourceToAcls(opts)
+      withAdminClient(opts) { adminClient =>
+        for ((resource, acls) <- resourceToAcl) {
+          val resourcePattern = resource.toPattern
+          println(s"Adding ACLs for resource `$resourcePattern`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          val aclBindings = acls.map(acl => new AclBinding(resourcePattern, getAccessControlEntry(acl))).asJavaCollection
+          adminClient.createAcls(aclBindings).all().get()
+        }
+
+        listAcls()
+      }
     }
-    finally CoreUtils.swallow(authZ.close(), this)
-  }
 
-  private def addAcl(opts: AclCommandOptions) {
-    val patternType: PatternType = opts.options.valueOf(opts.resourcePatternType)
-    if (!patternType.isSpecific)
-      CommandLineUtils.printUsageAndDie(opts.parser, s"A '--resource-pattern-type' value of '$patternType' is not valid when adding acls.")
+    def removeAcls(): Unit = {
+      withAdminClient(opts) { adminClient =>
+        val filterToAcl = getResourceFilterToAcls(opts)
+
+        for ((filter, acls) <- filterToAcl) {
+          if (acls.isEmpty) {
+            if (confirmAction(opts, s"Are you sure you want to delete all ACLs for resource filter `$filter`? (y/n)"))
+              removeAcls(adminClient, acls, filter)
+          } else {
+            if (confirmAction(opts, s"Are you sure you want to remove ACLs: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline from resource filter `$filter`? (y/n)"))
+              removeAcls(adminClient, acls, filter)
+          }
+        }
+
+        listAcls()
+      }
+    }
 
-    withAuthorizer(opts) { authorizer =>
-      val resourceToAcl = getResourceFilterToAcls(opts).map {
-        case (filter, acls) =>
-          Resource(ResourceType.fromJava(filter.resourceType()), filter.name(), filter.patternType()) -> acls
+    def listAcls(): Unit = {
+      withAdminClient(opts) { adminClient =>
+        val filters = getResourceFilter(opts, dieIfNoResourceFound = false)
+        val resourceToAcls = getAcls(adminClient, filters)
+
+        for ((resource, acls) <- resourceToAcls)
+          println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
       }
+    }
 
-      if (resourceToAcl.values.exists(_.isEmpty))
-        CommandLineUtils.printUsageAndDie(opts.parser, "You must specify one of: --allow-principal, --deny-principal when trying to add ACLs.")
+    private def getAccessControlEntry(acl: Acl): AccessControlEntry = {
+      new AccessControlEntry(acl.principal.toString, acl.host, acl.operation.toJava, acl.permissionType.toJava)
+    }
 
-      for ((resource, acls) <- resourceToAcl) {
-        println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
-        authorizer.addAcls(acls, resource)
+    private def removeAcls(adminClient: JAdminClient, acls: Set[Acl], filter: ResourcePatternFilter): Unit = {
+      if (acls.isEmpty)
+        adminClient.deleteAcls(List(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asJava).all().get()
+      else {
+        val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, getAccessControlEntryFilter(acl))).toList.asJava
+        adminClient.deleteAcls(aclBindingFilters).all().get()
       }
+    }
+
+    private def getAccessControlEntryFilter(acl: Acl): AccessControlEntryFilter = {
+      new AccessControlEntryFilter(acl.principal.toString, acl.host, acl.operation.toJava, acl.permissionType.toJava)
+    }
 
-      listAcl(opts)
+    private def getAcls(adminClient: JAdminClient, filters: Set[ResourcePatternFilter]): Map[ResourcePattern, Set[AccessControlEntry]] = {
+      val aclBindings =
+        if (filters.isEmpty) adminClient.describeAcls(AclBindingFilter.ANY).values().get().asScala.toList
+        else {
+          val results = for (filter <- filters) yield {
+            adminClient.describeAcls(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).values().get().asScala.toList
+          }
+          results.reduceLeft(_ ++ _)
+        }
+
+      val resourceToAcls = mutable.Map[ResourcePattern, Set[AccessControlEntry]]().withDefaultValue(Set())
+
+      aclBindings.foreach(aclBinding => resourceToAcls(aclBinding.pattern()) = resourceToAcls(aclBinding.pattern()) + aclBinding.entry())
+      resourceToAcls.toMap
     }
   }
 
-  private def removeAcl(opts: AclCommandOptions) {
-    withAuthorizer(opts) { authorizer =>
-      val filterToAcl = getResourceFilterToAcls(opts)
+  class AuthorizerService(val opts: AclCommandOptions) extends AclCommandService with Logging {
 
-      for ((filter, acls) <- filterToAcl) {
-        if (acls.isEmpty) {
-          if (confirmAction(opts, s"Are you sure you want to delete all ACLs for resource filter `$filter`? (y/n)"))
-            removeAcls(authorizer, acls, filter)
+    private def withAuthorizer()(f: Authorizer => Unit) {
+      val defaultProps = Map(KafkaConfig.ZkEnableSecureAclsProp -> JaasUtils.isZkSecurityEnabled)
+      val authorizerProperties =
+        if (opts.options.has(opts.authorizerPropertiesOpt)) {
+          val authorizerProperties = opts.options.valuesOf(opts.authorizerPropertiesOpt).asScala
+          defaultProps ++ CommandLineUtils.parseKeyValueArgs(authorizerProperties, acceptMissingValue = false).asScala
         } else {
-          if (confirmAction(opts, s"Are you sure you want to remove ACLs: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline from resource filter `$filter`? (y/n)"))
-            removeAcls(authorizer, acls, filter)
+          defaultProps
         }
+
+      val authorizerClass = if (opts.options.has(opts.authorizerOpt))
+        opts.options.valueOf(opts.authorizerOpt)
+      else
+        classOf[SimpleAclAuthorizer].getName
+
+      val authZ = CoreUtils.createObject[Authorizer](authorizerClass)
+      try {
+        authZ.configure(authorizerProperties.asJava)
+        f(authZ)
       }
+      finally CoreUtils.swallow(authZ.close(), this)
+    }
+
+    def addAcls(): Unit = {
+      val resourceToAcl = getResourceToAcls(opts)
+      withAuthorizer() { authorizer =>
+        for ((resource, acls) <- resourceToAcl) {
+          println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          authorizer.addAcls(acls, resource)
+        }
 
-      listAcl(opts)
+        listAcls()
+      }
     }
-  }
 
-  private def removeAcls(authorizer: Authorizer, acls: Set[Acl], filter: ResourcePatternFilter) {
-    getAcls(authorizer, filter)
-      .keys
-      .foreach(resource =>
-        if (acls.isEmpty) authorizer.removeAcls(resource)
-        else authorizer.removeAcls(acls, resource)
-      )
-  }
+    def removeAcls(): Unit = {
+      withAuthorizer() { authorizer =>
+        val filterToAcl = getResourceFilterToAcls(opts)
+
+        for ((filter, acls) <- filterToAcl) {
+          if (acls.isEmpty) {
+            if (confirmAction(opts, s"Are you sure you want to delete all ACLs for resource filter `$filter`? (y/n)"))
+              removeAcls(authorizer, acls, filter)
+          } else {
+            if (confirmAction(opts, s"Are you sure you want to remove ACLs: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline from resource filter `$filter`? (y/n)"))
+              removeAcls(authorizer, acls, filter)
+          }
+        }
 
-  private def listAcl(opts: AclCommandOptions) {
-    withAuthorizer(opts) { authorizer =>
-      val filters = getResourceFilter(opts, dieIfNoResourceFound = false)
+        listAcls()
+      }
+    }
+
+    def listAcls(): Unit = {
+      withAuthorizer() { authorizer =>
+        val filters = getResourceFilter(opts, dieIfNoResourceFound = false)
+
+        val resourceToAcls: Iterable[(Resource, Set[Acl])] =
+          if (filters.isEmpty) authorizer.getAcls()
+          else filters.flatMap(filter => getAcls(authorizer, filter))
 
-      val resourceToAcls: Iterable[(Resource, Set[Acl])] =
-        if (filters.isEmpty) authorizer.getAcls()
-        else filters.flatMap(filter => getAcls(authorizer, filter))
+        for ((resource, acls) <- resourceToAcls)
+          println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+      }
+    }
 
-      for ((resource, acls) <- resourceToAcls)
-        println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+    private def removeAcls(authorizer: Authorizer, acls: Set[Acl], filter: ResourcePatternFilter) {
+      getAcls(authorizer, filter)
+        .keys
+        .foreach(resource =>
+          if (acls.isEmpty) authorizer.removeAcls(resource)
+          else authorizer.removeAcls(acls, resource)
+        )
     }
+
+    private def getAcls(authorizer: Authorizer, filter: ResourcePatternFilter): Map[Resource, Set[Acl]] =
+      authorizer.getAcls()
+        .filter { case (resource, acl) => filter.matches(resource.toPattern) }
   }
 
-  private def getAcls(authorizer: Authorizer, filter: ResourcePatternFilter): Map[Resource, Set[Acl]] =
-    authorizer.getAcls()
-      .filter { case (resource, acl) => filter.matches(resource.toPattern) }
+  private def getResourceToAcls(opts: AclCommandOptions): Map[Resource, Set[Acl]] = {
+    val patternType: PatternType = opts.options.valueOf(opts.resourcePatternType)
+    if (!patternType.isSpecific)
+      CommandLineUtils.printUsageAndDie(opts.parser, s"A '--resource-pattern-type' value of '$patternType' is not valid when adding acls.")
+
+    val resourceToAcl = getResourceFilterToAcls(opts).map {
+      case (filter, acls) =>
+        Resource(ResourceType.fromJava(filter.resourceType()), filter.name(), filter.patternType()) -> acls
+    }
+
+    if (resourceToAcl.values.exists(_.isEmpty))
+      CommandLineUtils.printUsageAndDie(opts.parser, "You must specify one of: --allow-principal, --deny-principal when trying to add ACLs.")
+
+    resourceToAcl
+  }
 
   private def getResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[Acl]] = {
     var resourceToAcls = Map.empty[ResourcePatternFilter, Set[Acl]]
@@ -257,7 +381,7 @@ object AclCommand extends Logging {
 
   private def getPrincipals(opts: AclCommandOptions, principalOptionSpec: ArgumentAcceptingOptionSpec[String]): Set[KafkaPrincipal] = {
     if (opts.options.has(principalOptionSpec))
-      opts.options.valuesOf(principalOptionSpec).asScala.map(s => KafkaPrincipal.fromString(s.trim)).toSet
+      opts.options.valuesOf(principalOptionSpec).asScala.map(s => SecurityUtils.parseKafkaPrincipal(s.trim)).toSet
     else
       Set.empty[KafkaPrincipal]
   }
@@ -305,11 +429,23 @@ object AclCommand extends Logging {
 
   class AclCommandOptions(args: Array[String]) {
     val parser = new OptionParser(false)
+    val CommandConfigDoc = "A property file containing configs to be passed to Admin Client."
+
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", "A list of host/port pairs to use for establishing the connection to the Kafka cluster." +
+      " This list should be in the form host1:port1,host2:port2,... This config is required for acl management using admin client API.")
+      .withRequiredArg
+      .describedAs("server to connect to")
+      .ofType(classOf[String])
+
+    val commandConfigOpt = parser.accepts("command-config", CommandConfigDoc)
+      .withOptionalArg()
+      .describedAs("command-config")
+      .ofType(classOf[String])
+
     val authorizerOpt = parser.accepts("authorizer", "Fully qualified class name of the authorizer, defaults to kafka.security.auth.SimpleAclAuthorizer.")
       .withRequiredArg
       .describedAs("authorizer")
       .ofType(classOf[String])
-      .defaultsTo(classOf[SimpleAclAuthorizer].getName)
 
     val authorizerPropertiesOpt = parser.accepts("authorizer-properties", "REQUIRED: properties required to configure an instance of Authorizer. " +
       "These are key=val pairs. For the default authorizer the example values are: zookeeper.connect=localhost:2181")
@@ -410,7 +546,17 @@ object AclCommand extends Logging {
     val options = parser.parse(args: _*)
 
     def checkArgs() {
-      CommandLineUtils.checkRequiredArgs(parser, options, authorizerPropertiesOpt)
+      if (options.has(bootstrapServerOpt) && options.has(authorizerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "Only one of --bootstrap-server or --authorizer must be specified")
+
+      if (!options.has(bootstrapServerOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, authorizerPropertiesOpt)
+
+      if (options.has(commandConfigOpt) && !options.has(bootstrapServerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "The --command-config option can only be used with --bootstrap-server option")
+
+      if (options.has(authorizerPropertiesOpt) && options.has(bootstrapServerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "The --authorizer-properties option can only be used with --authorizer option")
 
       val actions = Seq(addOpt, removeOpt, listOpt).count(options.has)
       if (actions != 1)
diff --git a/core/src/main/scala/kafka/security/SecurityUtils.scala b/core/src/main/scala/kafka/security/SecurityUtils.scala
index 5d42871f66f85..311e195795d7e 100644
--- a/core/src/main/scala/kafka/security/SecurityUtils.scala
+++ b/core/src/main/scala/kafka/security/SecurityUtils.scala
@@ -22,8 +22,7 @@ import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding, AclBindingFi
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.requests.ApiError
 import org.apache.kafka.common.resource.ResourcePattern
-import org.apache.kafka.common.security.auth.KafkaPrincipal
-
+import org.apache.kafka.common.utils.SecurityUtils._
 import scala.util.{Failure, Success, Try}
 
 
@@ -32,7 +31,7 @@ object SecurityUtils {
   def convertToResourceAndAcl(filter: AclBindingFilter): Either[ApiError, (Resource, Acl)] = {
     (for {
       resourceType <- Try(ResourceType.fromJava(filter.patternFilter.resourceType))
-      principal <- Try(KafkaPrincipal.fromString(filter.entryFilter.principal))
+      principal <- Try(parseKafkaPrincipal(filter.entryFilter.principal))
       operation <- Try(Operation.fromJava(filter.entryFilter.operation))
       permissionType <- Try(PermissionType.fromJava(filter.entryFilter.permissionType))
       resource = Resource(resourceType, filter.patternFilter.name, filter.patternFilter.patternType)
diff --git a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
index 05f61897e6a70..d5535a50ab134 100644
--- a/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/AclCommandTest.scala
@@ -20,20 +20,24 @@ import java.util.Properties
 
 import kafka.admin.AclCommand.AclCommandOptions
 import kafka.security.auth._
-import kafka.server.KafkaConfig
+import kafka.server.{KafkaConfig, KafkaServer}
 import kafka.utils.{Exit, Logging, TestUtils}
 import kafka.zk.ZooKeeperTestHarness
 import org.apache.kafka.common.resource.PatternType
+import org.apache.kafka.common.network.ListenerName
+
 import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED}
-import org.apache.kafka.common.security.auth.KafkaPrincipal
-import org.junit.{Before, Test}
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.apache.kafka.common.utils.SecurityUtils
+import org.junit.{After, Before, Test}
 
 class AclCommandTest extends ZooKeeperTestHarness with Logging {
 
-  private val principal: KafkaPrincipal = KafkaPrincipal.fromString("User:test2")
-  private val Users = Set(KafkaPrincipal.fromString("User:CN=writeuser,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown"),
-    principal,
-    KafkaPrincipal.fromString("""User:CN=\#User with special chars in CN : (\, \+ \" \\ \< \> \; ')"""))
+  var servers: Seq[KafkaServer] = Seq()
+
+  private val principal: KafkaPrincipal = SecurityUtils.parseKafkaPrincipal("User:test2")
+  private val Users = Set(SecurityUtils.parseKafkaPrincipal("User:CN=writeuser,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown"),
+    principal, SecurityUtils.parseKafkaPrincipal("""User:CN=\#User with special chars in CN : (\, \+ \" \\ \< \> \; ')"""))
   private val Hosts = Set("host1", "host2")
   private val AllowHostCommand = Array("--allow-host", "host1", "--allow-host", "host2")
   private val DenyHostCommand = Array("--deny-host", "host1", "--deny-host", "host2")
@@ -87,6 +91,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
 
   private var brokerProps: Properties = _
   private var zkArgs: Array[String] = _
+  private var adminArgs: Array[String] = _
 
   @Before
   override def setUp(): Unit = {
@@ -94,33 +99,66 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
 
     brokerProps = TestUtils.createBrokerConfig(0, zkConnect)
     brokerProps.put(KafkaConfig.AuthorizerClassNameProp, "kafka.security.auth.SimpleAclAuthorizer")
+    brokerProps.put(SimpleAclAuthorizer.SuperUsersProp, "User:ANONYMOUS")
 
     zkArgs = Array("--authorizer-properties", "zookeeper.connect=" + zkConnect)
   }
 
+  @After
+  override def tearDown() {
+    TestUtils.shutdownServers(servers)
+    super.tearDown()
+  }
+
   @Test
-  def testAclCli() {
+  def testAclCliWithAuthorizer(): Unit = {
+    testAclCli(zkArgs)
+  }
+
+  @Test
+  def testAclCliWithAdminAPI(): Unit = {
+    createServer()
+    testAclCli(adminArgs)
+  }
+
+  private def createServer(): Unit = {
+    servers = Seq(TestUtils.createServer(KafkaConfig.fromProps(brokerProps)))
+    val listenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)
+    adminArgs = Array("--bootstrap-server", TestUtils.bootstrapServers(servers, listenerName))
+  }
+
+  private def testAclCli(cmdArgs: Array[String]) {
     for ((resources, resourceCmd) <- ResourceToCommand) {
       for (permissionType <- PermissionType.values) {
         val operationToCmd = ResourceToOperations(resources)
         val (acls, cmd) = getAclToCommand(permissionType, operationToCmd._1)
-          AclCommand.main(zkArgs ++ cmd ++ resourceCmd ++ operationToCmd._2 :+ "--add")
+          AclCommand.main(cmdArgs ++ cmd ++ resourceCmd ++ operationToCmd._2 :+ "--add")
           for (resource <- resources) {
             withAuthorizer() { authorizer =>
               TestUtils.waitAndVerifyAcls(acls, authorizer, resource)
             }
           }
 
-          testRemove(resources, resourceCmd, brokerProps)
+          testRemove(cmdArgs, resources, resourceCmd)
       }
     }
   }
 
   @Test
-  def testProducerConsumerCli() {
+  def testProducerConsumerCliWithAuthorizer(): Unit = {
+    testProducerConsumerCli(zkArgs)
+  }
+
+  @Test
+  def testProducerConsumerCliWithAdminAPI(): Unit = {
+    createServer()
+    testProducerConsumerCli(adminArgs)
+  }
+
+  private def testProducerConsumerCli(cmdArgs: Array[String]) {
     for ((cmd, resourcesToAcls) <- CmdToResourcesToAcl) {
       val resourceCommand: Array[String] = resourcesToAcls.keys.map(ResourceToCommand).foldLeft(Array[String]())(_ ++ _)
-      AclCommand.main(zkArgs ++ getCmd(Allow) ++ resourceCommand ++ cmd :+ "--add")
+      AclCommand.main(cmdArgs ++ getCmd(Allow) ++ resourceCommand ++ cmd :+ "--add")
       for ((resources, acls) <- resourcesToAcls) {
         for (resource <- resources) {
           withAuthorizer() { authorizer =>
@@ -128,15 +166,25 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
           }
         }
       }
-      testRemove(resourcesToAcls.keys.flatten.toSet, resourceCommand ++ cmd, brokerProps)
+      testRemove(cmdArgs, resourcesToAcls.keys.flatten.toSet, resourceCommand ++ cmd)
     }
   }
 
   @Test
-  def testAclsOnPrefixedResources(): Unit = {
+  def testAclsOnPrefixedResourcesWithAuthorizer(): Unit = {
+    testAclsOnPrefixedResources(zkArgs)
+  }
+
+  @Test
+  def testAclsOnPrefixedResourcesWithAdminAPI(): Unit = {
+    createServer()
+    testAclsOnPrefixedResources(adminArgs)
+  }
+
+  private def testAclsOnPrefixedResources(cmdArgs: Array[String]): Unit = {
     val cmd = Array("--allow-principal", principal.toString, "--producer", "--topic", "Test-", "--resource-pattern-type", "Prefixed")
 
-    AclCommand.main(zkArgs ++ cmd :+ "--add")
+    AclCommand.main(cmdArgs ++ cmd :+ "--add")
 
     withAuthorizer() { authorizer =>
       val writeAcl = Acl(principal, Allow, Acl.WildCardHost, Write)
@@ -145,7 +193,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
       TestUtils.waitAndVerifyAcls(Set(writeAcl, describeAcl, createAcl), authorizer, Resource(Topic, "Test-", PREFIXED))
     }
 
-    AclCommand.main(zkArgs ++ cmd :+ "--remove" :+ "--force")
+    AclCommand.main(cmdArgs ++ cmd :+ "--remove" :+ "--force")
 
     withAuthorizer() { authorizer =>
       TestUtils.waitAndVerifyAcls(Set.empty[Acl], authorizer, Resource(Cluster, "kafka-cluster", LITERAL))
@@ -156,7 +204,8 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
   @Test(expected = classOf[IllegalArgumentException])
   def testInvalidAuthorizerProperty() {
     val args = Array("--authorizer-properties", "zookeeper.connect " + zkConnect)
-    AclCommand.withAuthorizer(new AclCommandOptions(args))(null)
+    val aclCommandService = new AclCommand.AuthorizerService(new AclCommandOptions(args))
+    aclCommandService.listAcls()
   }
 
   @Test
@@ -188,9 +237,9 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     }
   }
 
-  private def testRemove(resources: Set[Resource], resourceCmd: Array[String], brokerProps: Properties) {
+  private def testRemove(cmdArgs: Array[String], resources: Set[Resource], resourceCmd: Array[String]) {
     for (resource <- resources) {
-      AclCommand.main(zkArgs ++ resourceCmd :+ "--remove" :+ "--force")
+      AclCommand.main(cmdArgs ++ resourceCmd :+ "--remove" :+ "--force")
       withAuthorizer() { authorizer =>
         TestUtils.waitAndVerifyAcls(Set.empty[Acl], authorizer, resource)
       }
@@ -208,7 +257,7 @@ class AclCommandTest extends ZooKeeperTestHarness with Logging {
     Users.foldLeft(cmd) ((cmd, user) => cmd ++ Array(principalCmd, user.toString))
   }
 
-  def withAuthorizer()(f: Authorizer => Unit) {
+  private def withAuthorizer()(f: Authorizer => Unit) {
     val kafkaConfig = KafkaConfig.fromProps(brokerProps, doLog = false)
     val authZ = new SimpleAclAuthorizer
     try {
diff --git a/docs/security.html b/docs/security.html
index d7859e08d72c5..e856a7e168658 100644
--- a/docs/security.html
+++ b/docs/security.html
@@ -1075,6 +1075,18 @@ 
    Command Line Interface
             
             Configuration
         
+        
+            --bootstrap-server
+            A list of host/port pairs to use for establishing the connection to the Kafka cluster. Only one of --bootstrap-server or --authorizer option must be specified.
+            
+            Configuration
+        
+        
+            --command-config
+            A property file containing configs to be passed to Admin Client. This option can only be used with --bootstrap-server option.
+            
+            Configuration
+        
         
             --cluster
             Indicates to the script that the user is trying to interact with acls on the singular cluster resource.
@@ -1199,7 +1211,17 @@ 
    Examples
             
     bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:Bob --consumer --topic Test-topic --group Group-1 
    
                 Note that for consumer option we must also specify the consumer group.
                 In order to remove a principal from producer or consumer role we just need to pass --remove option. 
    • 
-        
+
+        
  • AdminClient API based acl management
    
+            Users having Alter permission on ClusterResource can use AdminClient API for ACL management. kafka-acls.sh script supports AdminClient API to manage ACLs without interacting with zookeeper/authorizer directly.
+            All the above examples can be executed by using --bootstrap-server option. For example:
+
+            
    +            bin/kafka-acls.sh --bootstrap-server localhost:9092 --command-config /tmp/adminclient-configs.conf --add --allow-principal User:Bob --producer --topic Test-topic
+            bin/kafka-acls.sh --bootstrap-server localhost:9092 --command-config /tmp/adminclient-configs.conf --add --allow-principal User:Bob --consumer --topic Test-topic --group Group-1
+            bin/kafka-acls.sh --bootstrap-server localhost:9092 --command-config /tmp/adminclient-configs.conf --list --topic Test-topic
    • 
+
+    
 
     
    7.5 Incorporating Security Features in a Running Cluster
    
         You can secure a running cluster via one or more of the supported protocols discussed previously. This is done in phases:

From 4106ff6e5a61b76c5f05377df6f31fa8ae50c73b Mon Sep 17 00:00:00 2001
From: Manikumar Reddy O 
Date: Sat, 8 Sep 2018 20:13:22 +0530
Subject: [PATCH 0784/1847] MINOR: Enable topic deletion in test configs
 (#5616)

Reviewers: Ismael Juma 
---
 .../scala/unit/kafka/admin/DelegationTokenCommandTest.scala   | 2 +-
 core/src/test/scala/unit/kafka/metrics/MetricsTest.scala      | 2 +-
 core/src/test/scala/unit/kafka/server/BaseRequestTest.scala   | 2 +-
 .../scala/unit/kafka/server/DelegationTokenRequestsTest.scala | 2 +-
 core/src/test/scala/unit/kafka/utils/TestUtils.scala          | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala b/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala
index 6ae8f5e83ac3c..a4c27e502883f 100644
--- a/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala
@@ -49,7 +49,7 @@ class DelegationTokenCommandTest extends BaseRequestTest with SaslSetup {
 
   override def generateConfigs = {
     val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect,
-      enableControlledShutdown = false, enableDeleteTopic = true,
+      enableControlledShutdown = false,
       interBrokerSecurityProtocol = Some(securityProtocol),
       trustStoreFile = trustStoreFile, saslProperties = serverSaslProperties, enableToken = true)
     props.foreach(propertyOverrides)
diff --git a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
index e67fad14cede0..76cce0f1f6c42 100644
--- a/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
+++ b/core/src/test/scala/unit/kafka/metrics/MetricsTest.scala
@@ -42,7 +42,7 @@ class MetricsTest extends KafkaServerTestHarness with Logging {
   overridingProps.put(KafkaConfig.NumPartitionsProp, numParts.toString)
 
   def generateConfigs =
-    TestUtils.createBrokerConfigs(numNodes, zkConnect, enableDeleteTopic=true).map(KafkaConfig.fromProps(_, overridingProps))
+    TestUtils.createBrokerConfigs(numNodes, zkConnect).map(KafkaConfig.fromProps(_, overridingProps))
 
   val nMessages = 2
 
diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
index 4d178d76e9964..09ffe4fe83c24 100644
--- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala
@@ -43,7 +43,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness {
 
   override def generateConfigs = {
     val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect,
-      enableControlledShutdown = false, enableDeleteTopic = true,
+      enableControlledShutdown = false,
       interBrokerSecurityProtocol = Some(securityProtocol),
       trustStoreFile = trustStoreFile, saslProperties = serverSaslProperties, logDirCount = logDirCount)
     props.foreach(propertyOverrides)
diff --git a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala
index a00275084fbc3..6f79a9a20a6c6 100644
--- a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala
@@ -48,7 +48,7 @@ class DelegationTokenRequestsTest extends BaseRequestTest with SaslSetup {
 
   override def generateConfigs = {
     val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect,
-      enableControlledShutdown = false, enableDeleteTopic = true,
+      enableControlledShutdown = false,
       interBrokerSecurityProtocol = Some(securityProtocol),
       trustStoreFile = trustStoreFile, saslProperties = serverSaslProperties, enableToken = true)
     props.foreach(propertyOverrides)
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index d42d02c210d17..4dc822b52c9bd 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -150,7 +150,7 @@ object TestUtils extends Logging {
   def createBrokerConfigs(numConfigs: Int,
     zkConnect: String,
     enableControlledShutdown: Boolean = true,
-    enableDeleteTopic: Boolean = false,
+    enableDeleteTopic: Boolean = true,
     interBrokerSecurityProtocol: Option[SecurityProtocol] = None,
     trustStoreFile: Option[File] = None,
     saslProperties: Option[Properties] = None,
@@ -202,7 +202,7 @@ object TestUtils extends Logging {
   def createBrokerConfig(nodeId: Int,
                          zkConnect: String,
                          enableControlledShutdown: Boolean = true,
-                         enableDeleteTopic: Boolean = false,
+                         enableDeleteTopic: Boolean = true,
                          port: Int = RandomPort,
                          interBrokerSecurityProtocol: Option[SecurityProtocol] = None,
                          trustStoreFile: Option[File] = None,

From 0984a76b712caa18c688eafbacaa2a7c889d27b2 Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Sat, 8 Sep 2018 17:00:13 -0700
Subject: [PATCH 0785/1847] MINOR: Move common out of range handling into
 AbstractFetcherThread (#5608)

This patch removes the duplication of the out of range handling between `ReplicaFetcherThread` and `ReplicaAlterLogDirsThread` and attempts to expose a cleaner API for extension. It also adds a mock implementation to facilitate testing and several new test cases.

Reviewers: Jun Rao 
---
 .../kafka/common/requests/FetchRequest.java   |   2 +-
 .../main/scala/kafka/cluster/Partition.scala  |  20 +-
 .../main/scala/kafka/cluster/Replica.scala    |   5 +-
 .../kafka/server/AbstractFetcherThread.scala  | 231 ++++++---
 .../server/ReplicaAlterLogDirsThread.scala    |  79 +--
 .../kafka/server/ReplicaFetcherThread.scala   | 140 ++----
 .../ReplicaFetcherThreadFatalErrorTest.scala  |   2 +-
 .../server/AbstractFetcherThreadTest.scala    | 454 ++++++++++++++----
 8 files changed, 615 insertions(+), 318 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
index 6e25f7cef6659..e013f5ef0d2b1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
@@ -267,7 +267,7 @@ public static class Builder extends AbstractRequest.Builder {
         private IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED;
         private int maxBytes = DEFAULT_RESPONSE_MAX_BYTES;
         private FetchMetadata metadata = FetchMetadata.LEGACY;
-        private List toForget = Collections.emptyList();
+        private List toForget = Collections.emptyList();
 
         public static Builder forConsumer(int maxWait, int minBytes, Map fetchData) {
             return new Builder(ApiKeys.FETCH.oldestVersion(), ApiKeys.FETCH.latestVersion(),
diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index d76d6d06b5bdd..2036bb09da897 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -16,7 +16,6 @@
  */
 package kafka.cluster
 
-
 import java.util.concurrent.locks.ReentrantReadWriteLock
 
 import com.yammer.metrics.core.Gauge
@@ -591,26 +590,25 @@ class Partition(val topic: String,
     laggingReplicas
   }
 
-  private def doAppendRecordsToFollowerOrFutureReplica(records: MemoryRecords, isFuture: Boolean): Unit = {
+  private def doAppendRecordsToFollowerOrFutureReplica(records: MemoryRecords, isFuture: Boolean): Option[LogAppendInfo] = {
+    // The read lock is needed to handle race condition if request handler thread tries to
+    // remove future replica after receiving AlterReplicaLogDirsRequest.
     inReadLock(leaderIsrUpdateLock) {
       if (isFuture) {
-        // The read lock is needed to handle race condition if request handler thread tries to
-        // remove future replica after receiving AlterReplicaLogDirsRequest.
-        inReadLock(leaderIsrUpdateLock) {
-          getReplica(Request.FutureLocalReplicaId) match {
-            case Some(replica) => replica.log.get.appendAsFollower(records)
-            case None => // Future replica is removed by a non-ReplicaAlterLogDirsThread before this method is called
-          }
+        // Note the replica may be undefined if it is removed by a non-ReplicaAlterLogDirsThread before
+        // this method is called
+        getReplica(Request.FutureLocalReplicaId).map { replica =>
+          replica.log.get.appendAsFollower(records)
         }
       } else {
         // The read lock is needed to prevent the follower replica from being updated while ReplicaAlterDirThread
         // is executing maybeDeleteAndSwapFutureReplica() to replace follower replica with the future replica.
-        getReplicaOrException().log.get.appendAsFollower(records)
+        Some(getReplicaOrException().log.get.appendAsFollower(records))
       }
     }
   }
 
-  def appendRecordsToFollowerOrFutureReplica(records: MemoryRecords, isFuture: Boolean) {
+  def appendRecordsToFollowerOrFutureReplica(records: MemoryRecords, isFuture: Boolean): Option[LogAppendInfo] = {
     try {
       doAppendRecordsToFollowerOrFutureReplica(records, isFuture)
     } catch {
diff --git a/core/src/main/scala/kafka/cluster/Replica.scala b/core/src/main/scala/kafka/cluster/Replica.scala
index 962aaffdb08b3..839579bccb22c 100644
--- a/core/src/main/scala/kafka/cluster/Replica.scala
+++ b/core/src/main/scala/kafka/cluster/Replica.scala
@@ -18,6 +18,7 @@
 package kafka.cluster
 
 import kafka.log.Log
+import kafka.server.epoch.LeaderEpochCache
 import kafka.utils.Logging
 import kafka.server.{LogOffsetMetadata, LogReadResult}
 import org.apache.kafka.common.{KafkaException, TopicPartition}
@@ -52,9 +53,9 @@ class Replica(val brokerId: Int,
 
   def isLocal: Boolean = log.isDefined
 
-  def lastCaughtUpTimeMs = _lastCaughtUpTimeMs
+  def lastCaughtUpTimeMs: Long = _lastCaughtUpTimeMs
 
-  val epochs = log.map(_.leaderEpochCache)
+  val epochs: Option[LeaderEpochCache] = log.map(_.leaderEpochCache)
 
   info(s"Replica loaded for partition $topicPartition with initial high watermark $initialHighWatermarkValue")
   log.foreach(_.onHighWatermarkIncremented(initialHighWatermarkValue))
diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
index e753f6e2fdc1b..44137cf35c3b1 100755
--- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala
@@ -20,7 +20,7 @@ package kafka.server
 import java.nio.ByteBuffer
 import java.util.concurrent.locks.ReentrantLock
 
-import kafka.cluster.{BrokerEndPoint, Replica}
+import kafka.cluster.BrokerEndPoint
 import kafka.utils.{DelayedItem, Pool, ShutdownableThread}
 import org.apache.kafka.common.errors.{CorruptRecordException, KafkaStorageException}
 import org.apache.kafka.common.requests.EpochEndOffset._
@@ -36,10 +36,11 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicLong
 
 import com.yammer.metrics.core.Gauge
+import kafka.log.LogAppendInfo
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.apache.kafka.common.internals.{FatalExitError, PartitionStates}
 import org.apache.kafka.common.record.{FileRecords, MemoryRecords, Records}
-import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, FetchResponse}
+import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, FetchResponse, ListOffsetRequest}
 
 import scala.math._
 
@@ -50,8 +51,7 @@ abstract class AbstractFetcherThread(name: String,
                                      clientId: String,
                                      val sourceBroker: BrokerEndPoint,
                                      fetchBackOffMs: Int = 0,
-                                     isInterruptible: Boolean = true,
-                                     includeLogTruncation: Boolean)
+                                     isInterruptible: Boolean = true)
   extends ShutdownableThread(name, isInterruptible) {
 
   type PD = FetchResponse.PartitionData[Records]
@@ -67,21 +67,31 @@ abstract class AbstractFetcherThread(name: String,
   /* callbacks to be defined in subclass */
 
   // process fetched data
-  protected def processPartitionData(topicPartition: TopicPartition, fetchOffset: Long, partitionData: PD,
-                                     records: MemoryRecords)
+  protected def processPartitionData(topicPartition: TopicPartition,
+                                     fetchOffset: Long,
+                                     partitionData: PD): Option[LogAppendInfo]
 
-  // handle a partition whose offset is out of range and return a new fetch offset
-  protected def handleOffsetOutOfRange(topicPartition: TopicPartition): Long
+  protected def truncate(topicPartition: TopicPartition, truncationState: OffsetTruncationState): Unit
 
-  protected def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset]
-
-  protected def truncate(topicPartition: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState
+  protected def truncateFullyAndStartAt(topicPartition: TopicPartition, offset: Long): Unit
 
   protected def buildFetch(partitionMap: Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]]
 
-  protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)]
+  protected def isUncleanLeaderElectionAllowed(topicPartition: TopicPartition): Boolean
+
+  protected def latestEpoch(topicPartition: TopicPartition): Option[Int]
+
+  protected def logEndOffset(topicPartition: TopicPartition): Long
+
+  protected def endOffsetForEpoch(topicPartition: TopicPartition, epoch: Int): Option[OffsetAndEpoch]
+
+  protected def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset]
+
+  protected def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)]
+
+  protected def fetchEarliestOffsetFromLeader(topicPartition: TopicPartition): Long
 
-  protected def getReplica(tp: TopicPartition): Option[Replica]
+  protected def fetchLatestOffsetFromLeader(topicPartition: TopicPartition): Long
 
   override def shutdown() {
     initiateShutdown()
@@ -97,7 +107,10 @@ abstract class AbstractFetcherThread(name: String,
 
   override def doWork() {
     maybeTruncate()
+    maybeFetch()
+  }
 
+  private def maybeFetch(): Unit = {
     val (fetchStates, fetchRequestOpt) = inLock(partitionMapLock) {
       val fetchStates = partitionStates.partitionStateMap.asScala
       val ResultWithPartitions(fetchRequestOpt, partitionsWithError) = buildFetch(fetchStates)
@@ -134,7 +147,7 @@ abstract class AbstractFetcherThread(name: String,
     partitionStates.partitionStates.asScala.foreach { state =>
       val tp = state.topicPartition
       if (state.value.isTruncatingLog) {
-        getReplica(tp).flatMap(_.epochs).map(_.latestEpoch) match {
+        latestEpoch(tp) match {
           case Some(latestEpoch) => partitionsWithEpochs += tp -> latestEpoch
           case None => partitionsWithoutEpochs += tp
         }
@@ -194,6 +207,12 @@ abstract class AbstractFetcherThread(name: String,
     ResultWithPartitions(fetchOffsets, partitionsWithError)
   }
 
+  private def truncate(topicPartition: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
+    val offsetTruncationState = getOffsetTruncationState(topicPartition, epochEndOffset)
+    truncate(topicPartition, offsetTruncationState)
+    offsetTruncationState
+  }
+
   private def processFetchRequest(fetchStates: Map[TopicPartition, PartitionFetchState],
                                   fetchRequest: FetchRequest.Builder): Unit = {
     val partitionsWithError = mutable.Set[TopicPartition]()
@@ -201,7 +220,7 @@ abstract class AbstractFetcherThread(name: String,
 
     try {
       trace(s"Sending fetch request $fetchRequest")
-      responseData = fetch(fetchRequest)
+      responseData = fetchFromLeader(fetchRequest)
     } catch {
       case t: Throwable =>
         if (isRunning) {
@@ -220,7 +239,6 @@ abstract class AbstractFetcherThread(name: String,
     if (responseData.nonEmpty) {
       // process fetched data
       inLock(partitionMapLock) {
-
         responseData.foreach { case (topicPartition, partitionData) =>
           Option(partitionStates.stateValue(topicPartition)).foreach { currentPartitionFetchState =>
             // It's possible that a partition is removed and re-added or truncated when there is a pending fetch request.
@@ -231,28 +249,31 @@ abstract class AbstractFetcherThread(name: String,
               partitionData.error match {
                 case Errors.NONE =>
                   try {
-                    val records = toMemoryRecords(partitionData.records)
-                    val newOffset = records.batches.asScala.lastOption.map(_.nextOffset).getOrElse(
-                      currentPartitionFetchState.fetchOffset)
-
-                    fetcherLagStats.getAndMaybePut(topicPartition).lag = Math.max(0L, partitionData.highWatermark - newOffset)
                     // Once we hand off the partition data to the subclass, we can't mess with it any more in this thread
-                    processPartitionData(topicPartition, currentPartitionFetchState.fetchOffset, partitionData, records)
-
-                    val validBytes = records.validBytes
-                    // ReplicaDirAlterThread may have removed topicPartition from the partitionStates after processing the partition data
-                    if (validBytes > 0 && partitionStates.contains(topicPartition)) {
-                      // Update partitionStates only if there is no exception during processPartitionData
-                      partitionStates.updateAndMoveToEnd(topicPartition, new PartitionFetchState(newOffset))
-                      fetcherStats.byteRate.mark(validBytes)
+                    val logAppendInfoOpt = processPartitionData(topicPartition, currentPartitionFetchState.fetchOffset,
+                      partitionData)
+
+                    logAppendInfoOpt.foreach { logAppendInfo =>
+                      val nextOffset = logAppendInfo.lastOffset + 1
+                      fetcherLagStats.getAndMaybePut(topicPartition).lag = Math.max(0L, partitionData.highWatermark - nextOffset)
+
+                      val validBytes = logAppendInfo.validBytes
+                      // ReplicaDirAlterThread may have removed topicPartition from the partitionStates after processing the partition data
+                      if (validBytes > 0 && partitionStates.contains(topicPartition)) {
+                        // Update partitionStates only if there is no exception during processPartitionData
+                        partitionStates.updateAndMoveToEnd(topicPartition, new PartitionFetchState(nextOffset))
+                        fetcherStats.byteRate.mark(validBytes)
+                      }
                     }
                   } catch {
                     case ime: CorruptRecordException =>
                       // we log the error and continue. This ensures two things
-                      // 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread down and cause other topic partition to also lag
-                      // 2. If the message is corrupt due to a transient state in the log (truncation, partial writes can cause this), we simply continue and
-                      // should get fixed in the subsequent fetches
-                      error(s"Found invalid messages during fetch for partition $topicPartition offset ${currentPartitionFetchState.fetchOffset}", ime)
+                      // 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread
+                      //    down and cause other topic partition to also lag
+                      // 2. If the message is corrupt due to a transient state in the log (truncation, partial writes
+                      //    can cause this), we simply continue and should get fixed in the subsequent fetches
+                      error(s"Found invalid messages during fetch for partition $topicPartition " +
+                        s"offset ${currentPartitionFetchState.fetchOffset}", ime)
                       partitionsWithError += topicPartition
                     case e: KafkaStorageException =>
                       error(s"Error while processing data for partition $topicPartition", e)
@@ -297,8 +318,6 @@ abstract class AbstractFetcherThread(name: String,
   }
 
   def markPartitionsForTruncation(topicPartition: TopicPartition, truncationOffset: Long) {
-    if (!includeLogTruncation)
-      throw new IllegalStateException("Truncation should not be requested if includeLogTruncation is disabled")
     partitionMapLock.lockInterruptibly()
     try {
       Option(partitionStates.stateValue(topicPartition)).foreach { state =>
@@ -318,9 +337,9 @@ abstract class AbstractFetcherThread(name: String,
       }.map { case (tp, initialFetchOffset) =>
         val fetchState =
           if (initialFetchOffset < 0)
-            new PartitionFetchState(handleOffsetOutOfRange(tp), includeLogTruncation)
+            new PartitionFetchState(handleOffsetOutOfRange(tp), truncatingLog = true)
           else
-            new PartitionFetchState(initialFetchOffset, includeLogTruncation)
+            new PartitionFetchState(initialFetchOffset, truncatingLog = true)
         tp -> fetchState
       }
 
@@ -341,11 +360,11 @@ abstract class AbstractFetcherThread(name: String,
   private def updateFetchOffsetAndMaybeMarkTruncationComplete(fetchOffsets: Map[TopicPartition, OffsetTruncationState]) {
     val newStates: Map[TopicPartition, PartitionFetchState] = partitionStates.partitionStates.asScala
       .map { state =>
-        val maybeTruncationComplete = fetchOffsets.get(state.topicPartition()) match {
+        val maybeTruncationComplete = fetchOffsets.get(state.topicPartition) match {
           case Some(offsetTruncationState) => PartitionFetchState(offsetTruncationState.offset, state.value.delay, truncatingLog = !offsetTruncationState.truncationCompleted)
           case None => state.value()
         }
-        (state.topicPartition(), maybeTruncationComplete)
+        (state.topicPartition, maybeTruncationComplete)
       }.toMap
     partitionStates.set(newStates.asJava)
   }
@@ -372,57 +391,121 @@ abstract class AbstractFetcherThread(name: String,
    *
    * @param tp                    Topic partition
    * @param leaderEpochOffset     Epoch end offset received from the leader for this topic partition
-   * @param replica               Follower's replica, which is either local replica
-   *                              (ReplicaFetcherThread) or future replica (ReplicaAlterLogDirsThread)
-   * @param isFutureReplica       true if called from ReplicaAlterLogDirsThread
    */
-  def getOffsetTruncationState(tp: TopicPartition, leaderEpochOffset: EpochEndOffset, replica: Replica, isFutureReplica: Boolean = false): OffsetTruncationState = {
-    // to make sure we can distinguish log output for fetching from remote leader or local replica
-    val followerName = if (isFutureReplica) "future replica" else "follower"
-
+  private def getOffsetTruncationState(tp: TopicPartition, leaderEpochOffset: EpochEndOffset): OffsetTruncationState = {
     if (leaderEpochOffset.endOffset == UNDEFINED_EPOCH_OFFSET) {
       // truncate to initial offset which is the high watermark for follower replica. For
       // future replica, it is either high watermark of the future replica or current
       // replica's truncation offset (when the current replica truncates, it forces future
       // replica's partition state to 'truncating' and sets initial offset to its truncation offset)
-      warn(s"Based on $followerName's leader epoch, leader replied with an unknown offset in ${replica.topicPartition}. " +
+      warn(s"Based on replica's leader epoch, leader replied with an unknown offset in $tp. " +
            s"The initial fetch offset ${partitionStates.stateValue(tp).fetchOffset} will be used for truncation.")
       OffsetTruncationState(partitionStates.stateValue(tp).fetchOffset, truncationCompleted = true)
     } else if (leaderEpochOffset.leaderEpoch == UNDEFINED_EPOCH) {
       // either leader or follower or both use inter-broker protocol version < KAFKA_2_0_IV0
       // (version 0 of OffsetForLeaderEpoch request/response)
-      warn(s"Leader or $followerName is on protocol version where leader epoch is not considered in the OffsetsForLeaderEpoch response. " +
-           s"The leader's offset ${leaderEpochOffset.endOffset} will be used for truncation in ${replica.topicPartition}.")
-      OffsetTruncationState(min(leaderEpochOffset.endOffset, replica.logEndOffset.messageOffset), truncationCompleted = true)
+      warn(s"Leader or replica is on protocol version where leader epoch is not considered in the OffsetsForLeaderEpoch response. " +
+           s"The leader's offset ${leaderEpochOffset.endOffset} will be used for truncation in $tp.")
+      OffsetTruncationState(min(leaderEpochOffset.endOffset, logEndOffset(tp)), truncationCompleted = true)
     } else {
+      val replicaEndOffset = logEndOffset(tp)
+
       // get (leader epoch, end offset) pair that corresponds to the largest leader epoch
       // less than or equal to the requested epoch.
-      val (followerEpoch, followerEndOffset) = replica.epochs.get.endOffsetFor(leaderEpochOffset.leaderEpoch)
-      if (followerEndOffset == UNDEFINED_EPOCH_OFFSET) {
-        // This can happen if the follower was not tracking leader epochs at that point (before the
-        // upgrade, or if this broker is new). Since the leader replied with epoch <
-        // requested epoch from follower, so should be safe to truncate to leader's
-        // offset (this is the same behavior as post-KIP-101 and pre-KIP-279)
-        warn(s"Based on $followerName's leader epoch, leader replied with epoch ${leaderEpochOffset.leaderEpoch} " +
-             s"below any $followerName's tracked epochs for ${replica.topicPartition}. " +
-             s"The leader's offset only ${leaderEpochOffset.endOffset} will be used for truncation.")
-        OffsetTruncationState(min(leaderEpochOffset.endOffset, replica.logEndOffset.messageOffset), truncationCompleted = true)
-      } else if (followerEpoch != leaderEpochOffset.leaderEpoch) {
-        // the follower does not know about the epoch that leader replied with
-        // we truncate to the end offset of the largest epoch that is smaller than the
-        // epoch the leader replied with, and send another offset for leader epoch request
-        val intermediateOffsetToTruncateTo = min(followerEndOffset, replica.logEndOffset.messageOffset)
-        info(s"Based on $followerName's leader epoch, leader replied with epoch ${leaderEpochOffset.leaderEpoch} " +
-             s"unknown to the $followerName for ${replica.topicPartition}. " +
-             s"Will truncate to $intermediateOffsetToTruncateTo and send another leader epoch request to the leader.")
-        OffsetTruncationState(intermediateOffsetToTruncateTo, truncationCompleted = false)
-      } else {
-        val offsetToTruncateTo = min(followerEndOffset, leaderEpochOffset.endOffset)
-        OffsetTruncationState(min(offsetToTruncateTo, replica.logEndOffset.messageOffset), truncationCompleted = true)
+      endOffsetForEpoch(tp, leaderEpochOffset.leaderEpoch) match {
+        case Some(OffsetAndEpoch(followerEndOffset, followerEpoch)) =>
+          if (followerEpoch != leaderEpochOffset.leaderEpoch) {
+            // the follower does not know about the epoch that leader replied with
+            // we truncate to the end offset of the largest epoch that is smaller than the
+            // epoch the leader replied with, and send another offset for leader epoch request
+            val intermediateOffsetToTruncateTo = min(followerEndOffset, replicaEndOffset)
+            info(s"Based on replica's leader epoch, leader replied with epoch ${leaderEpochOffset.leaderEpoch} " +
+              s"unknown to the replica for $tp. " +
+              s"Will truncate to $intermediateOffsetToTruncateTo and send another leader epoch request to the leader.")
+            OffsetTruncationState(intermediateOffsetToTruncateTo, truncationCompleted = false)
+          } else {
+            val offsetToTruncateTo = min(followerEndOffset, leaderEpochOffset.endOffset)
+            OffsetTruncationState(min(offsetToTruncateTo, replicaEndOffset), truncationCompleted = true)
+          }
+        case None =>
+          // This can happen if the follower was not tracking leader epochs at that point (before the
+          // upgrade, or if this broker is new). Since the leader replied with epoch <
+          // requested epoch from follower, so should be safe to truncate to leader's
+          // offset (this is the same behavior as post-KIP-101 and pre-KIP-279)
+          warn(s"Based on replica's leader epoch, leader replied with epoch ${leaderEpochOffset.leaderEpoch} " +
+            s"below any replica's tracked epochs for $tp. " +
+            s"The leader's offset only ${leaderEpochOffset.endOffset} will be used for truncation.")
+          OffsetTruncationState(min(leaderEpochOffset.endOffset, replicaEndOffset), truncationCompleted = true)
       }
     }
   }
 
+  /**
+   * Handle a partition whose offset is out of range and return a new fetch offset.
+   */
+  protected def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = {
+    val replicaEndOffset = logEndOffset(topicPartition)
+
+    /**
+     * Unclean leader election: A follower goes down, in the meanwhile the leader keeps appending messages. The follower comes back up
+     * and before it has completely caught up with the leader's logs, all replicas in the ISR go down. The follower is now uncleanly
+     * elected as the new leader, and it starts appending messages from the client. The old leader comes back up, becomes a follower
+     * and it may discover that the current leader's end offset is behind its own end offset.
+     *
+     * In such a case, truncate the current follower's log to the current leader's end offset and continue fetching.
+     *
+     * There is a potential for a mismatch between the logs of the two replicas here. We don't fix this mismatch as of now.
+     */
+    val leaderEndOffset = fetchLatestOffsetFromLeader(topicPartition)
+    if (leaderEndOffset < replicaEndOffset) {
+      // Prior to truncating the follower's log, ensure that doing so is not disallowed by the configuration for unclean leader election.
+      // This situation could only happen if the unclean election configuration for a topic changes while a replica is down. Otherwise,
+      // we should never encounter this situation since a non-ISR leader cannot be elected if disallowed by the broker configuration.
+      if (!isUncleanLeaderElectionAllowed(topicPartition)) {
+        // Log a fatal error and shutdown the broker to ensure that data loss does not occur unexpectedly.
+        fatal(s"Exiting because log truncation is not allowed for partition $topicPartition, current leader's " +
+          s"latest offset $leaderEndOffset is less than replica's latest offset $replicaEndOffset}")
+        throw new FatalExitError
+      }
+
+      warn(s"Reset fetch offset for partition $topicPartition from $replicaEndOffset to current " +
+        s"leader's latest offset $leaderEndOffset")
+      truncate(topicPartition, new EpochEndOffset(Errors.NONE, UNDEFINED_EPOCH, leaderEndOffset))
+      leaderEndOffset
+    } else {
+      /**
+       * If the leader's log end offset is greater than the follower's log end offset, there are two possibilities:
+       * 1. The follower could have been down for a long time and when it starts up, its end offset could be smaller than the leader's
+       * start offset because the leader has deleted old logs (log.logEndOffset < leaderStartOffset).
+       * 2. When unclean leader election occurs, it is possible that the old leader's high watermark is greater than
+       * the new leader's log end offset. So when the old leader truncates its offset to its high watermark and starts
+       * to fetch from the new leader, an OffsetOutOfRangeException will be thrown. After that some more messages are
+       * produced to the new leader. While the old leader is trying to handle the OffsetOutOfRangeException and query
+       * the log end offset of the new leader, the new leader's log end offset becomes higher than the follower's log end offset.
+       *
+       * In the first case, the follower's current log end offset is smaller than the leader's log start offset. So the
+       * follower should truncate all its logs, roll out a new segment and start to fetch from the current leader's log
+       * start offset.
+       * In the second case, the follower should just keep the current log segments and retry the fetch. In the second
+       * case, there will be some inconsistency of data between old and new leader. We are not solving it here.
+       * If users want to have strong consistency guarantees, appropriate configurations needs to be set for both
+       * brokers and producers.
+       *
+       * Putting the two cases together, the follower should fetch from the higher one of its replica log end offset
+       * and the current leader's log start offset.
+       */
+      val leaderStartOffset = fetchEarliestOffsetFromLeader(topicPartition)
+      warn(s"Reset fetch offset for partition $topicPartition from $replicaEndOffset to current " +
+        s"leader's start offset $leaderStartOffset")
+      val offsetToFetch = Math.max(leaderStartOffset, replicaEndOffset)
+      // Only truncate log when current leader's log start offset is greater than follower's log end offset.
+      if (leaderStartOffset > replicaEndOffset)
+        truncateFullyAndStartAt(topicPartition, leaderStartOffset)
+      offsetToFetch
+    }
+  }
+
+
   def delayPartitions(partitions: Iterable[TopicPartition], delay: Long) {
     partitionMapLock.lockInterruptibly()
     try {
@@ -459,7 +542,7 @@ abstract class AbstractFetcherThread(name: String,
     }.toMap
   }
 
-  private def toMemoryRecords(records: Records): MemoryRecords = {
+  protected def toMemoryRecords(records: Records): MemoryRecords = {
     records match {
       case r: MemoryRecords => r
       case r: FileRecords =>
@@ -587,3 +670,5 @@ case class OffsetTruncationState(offset: Long, truncationCompleted: Boolean) {
 
   override def toString = "offset:%d-truncationCompleted:%b".format(offset, truncationCompleted)
 }
+
+case class OffsetAndEpoch(offset: Long, epoch: Int)
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 16212018b58a5..5aec7a90d2922 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -20,13 +20,14 @@ package kafka.server
 import java.util
 
 import kafka.api.Request
-import kafka.cluster.{BrokerEndPoint, Replica}
+import kafka.cluster.BrokerEndPoint
+import kafka.log.LogAppendInfo
 import kafka.server.AbstractFetcherThread.ResultWithPartitions
 import kafka.server.QuotaFactory.UnboundedQuota
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.KafkaStorageException
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
-import org.apache.kafka.common.record.{MemoryRecords, Records}
+import org.apache.kafka.common.record.Records
 import org.apache.kafka.common.requests.EpochEndOffset._
 import org.apache.kafka.common.requests.FetchResponse.PartitionData
 import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest, FetchResponse}
@@ -44,18 +45,32 @@ class ReplicaAlterLogDirsThread(name: String,
                                 clientId = name,
                                 sourceBroker = sourceBroker,
                                 fetchBackOffMs = brokerConfig.replicaFetchBackoffMs,
-                                isInterruptible = false,
-                                includeLogTruncation = true) {
+                                isInterruptible = false) {
 
   private val replicaId = brokerConfig.brokerId
   private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes
   private val fetchSize = brokerConfig.replicaFetchMaxBytes
 
-  protected def getReplica(tp: TopicPartition): Option[Replica] = {
-    replicaMgr.getReplica(tp, Request.FutureLocalReplicaId)
+  override protected def latestEpoch(topicPartition: TopicPartition): Option[Int] = {
+    replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId).epochs.map(_.latestEpoch)
   }
 
-  def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
+  override protected def logEndOffset(topicPartition: TopicPartition): Long = {
+    replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId).logEndOffset.messageOffset
+  }
+
+  override protected def endOffsetForEpoch(topicPartition: TopicPartition, epoch: Int): Option[OffsetAndEpoch] = {
+    val replica = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId)
+    replica.epochs.flatMap { epochCache =>
+      val (foundEpoch, foundOffset) = epochCache.endOffsetFor(epoch)
+      if (foundOffset == UNDEFINED_EPOCH_OFFSET)
+        None
+      else
+        Some(OffsetAndEpoch(foundOffset, foundEpoch))
+    }
+  }
+
+  def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
     var partitionData: Seq[(TopicPartition, FetchResponse.PartitionData[Records])] = null
     val request = fetchRequest.build()
 
@@ -86,16 +101,18 @@ class ReplicaAlterLogDirsThread(name: String,
   }
 
   // process fetched data
-  def processPartitionData(topicPartition: TopicPartition, fetchOffset: Long, partitionData: PartitionData[Records],
-                           records: MemoryRecords) {
+  override def processPartitionData(topicPartition: TopicPartition,
+                                    fetchOffset: Long,
+                                    partitionData: PartitionData[Records]): Option[LogAppendInfo] = {
     val futureReplica = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId)
     val partition = replicaMgr.getPartition(topicPartition).get
+    val records = toMemoryRecords(partitionData.records)
 
     if (fetchOffset != futureReplica.logEndOffset.messageOffset)
       throw new IllegalStateException("Offset mismatch for the future replica %s: fetched offset = %d, log end offset = %d.".format(
         topicPartition, fetchOffset, futureReplica.logEndOffset.messageOffset))
 
-    partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = true)
+    val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = true)
     val futureReplicaHighWatermark = futureReplica.logEndOffset.messageOffset.min(partitionData.highWatermark)
     futureReplica.highWatermark = new LogOffsetMetadata(futureReplicaHighWatermark)
     futureReplica.maybeIncrementLogStartOffset(partitionData.logStartOffset)
@@ -104,29 +121,17 @@ class ReplicaAlterLogDirsThread(name: String,
       removePartitions(Set(topicPartition))
 
     quota.record(records.sizeInBytes)
+    logAppendInfo
   }
 
-  def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = {
-    val futureReplica = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId)
-    val currentReplica = replicaMgr.getReplicaOrException(topicPartition)
-    val partition = replicaMgr.getPartition(topicPartition).get
-    val logEndOffset: Long = currentReplica.logEndOffset.messageOffset
+  override protected def isUncleanLeaderElectionAllowed(topicPartition: TopicPartition): Boolean = true
 
-    if (logEndOffset < futureReplica.logEndOffset.messageOffset) {
-      warn("Future replica for partition %s reset its fetch offset from %d to current replica's latest offset %d"
-        .format(topicPartition, futureReplica.logEndOffset.messageOffset, logEndOffset))
-      partition.truncateTo(logEndOffset, isFuture = true)
-      logEndOffset
-    } else {
-      val currentReplicaStartOffset: Long = currentReplica.logStartOffset
-      warn("Future replica for partition %s reset its fetch offset from %d to current replica's start offset %d"
-        .format(topicPartition, futureReplica.logEndOffset.messageOffset, currentReplicaStartOffset))
-      val offsetToFetch = Math.max(currentReplicaStartOffset, futureReplica.logEndOffset.messageOffset)
-      // Only truncate the log when current replica's log start offset is greater than future replica's log end offset.
-      if (currentReplicaStartOffset > futureReplica.logEndOffset.messageOffset)
-        partition.truncateFullyAndStartAt(currentReplicaStartOffset, isFuture = true)
-      offsetToFetch
-    }
+  override protected def fetchEarliestOffsetFromLeader(topicPartition: TopicPartition): Long = {
+    replicaMgr.getReplicaOrException(topicPartition).logStartOffset
+  }
+
+  override protected def fetchLatestOffsetFromLeader(topicPartition: TopicPartition): Long = {
+    replicaMgr.getReplicaOrException(topicPartition).logEndOffset.messageOffset
   }
 
   /**
@@ -134,7 +139,7 @@ class ReplicaAlterLogDirsThread(name: String,
    * @param partitions map of topic partition -> leader epoch of the future replica
    * @return map of topic partition -> end offset for a requested leader epoch
    */
-  def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
+  override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
     partitions.map { case (tp, epoch) =>
       try {
         val (leaderEpoch, leaderOffset) = replicaMgr.getReplicaOrException(tp).epochs.get.endOffsetFor(epoch)
@@ -161,14 +166,14 @@ class ReplicaAlterLogDirsThread(name: String,
    * the future replica may miss "mark for truncation" event and must use the offset for leader epoch
    * exchange with the current replica to truncate to the largest common log prefix for the topic partition
    */
-  override def truncate(topicPartition: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
-    val futureReplica = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId)
+  override def truncate(topicPartition: TopicPartition, truncationState: OffsetTruncationState): Unit = {
     val partition = replicaMgr.getPartition(topicPartition).get
+    partition.truncateTo(truncationState.offset, isFuture = true)
+  }
 
-    val offsetTruncationState = getOffsetTruncationState(topicPartition, epochEndOffset, futureReplica,
-      isFutureReplica = true)
-    partition.truncateTo(offsetTruncationState.offset, isFuture = true)
-    offsetTruncationState
+  override protected def truncateFullyAndStartAt(topicPartition: TopicPartition, offset: Long): Unit = {
+    val partition = replicaMgr.getPartition(topicPartition).get
+    partition.truncateFullyAndStartAt(offset, isFuture = true)
   }
 
   def buildFetch(partitionMap: Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index 5624e848cfc52..1848eb741c381 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -18,14 +18,13 @@
 package kafka.server
 
 import kafka.api._
-import kafka.cluster.{BrokerEndPoint, Replica}
-import kafka.log.LogConfig
+import kafka.cluster.BrokerEndPoint
+import kafka.log.{LogAppendInfo, LogConfig}
 import kafka.server.AbstractFetcherThread.ResultWithPartitions
 import kafka.zk.AdminZkClient
 import org.apache.kafka.clients.FetchSessionHandler
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.KafkaStorageException
-import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
 import org.apache.kafka.common.record.{MemoryRecords, Records}
@@ -49,8 +48,7 @@ class ReplicaFetcherThread(name: String,
                                 clientId = name,
                                 sourceBroker = sourceBroker,
                                 fetchBackOffMs = brokerConfig.replicaFetchBackoffMs,
-                                isInterruptible = false,
-                                includeLogTruncation = true) {
+                                isInterruptible = false) {
 
   private val replicaId = brokerConfig.brokerId
   private val logContext = new LogContext(s"[ReplicaFetcher replicaId=$replicaId, leaderId=${sourceBroker.id}, " +
@@ -88,12 +86,26 @@ class ReplicaFetcherThread(name: String,
   private val minBytes = brokerConfig.replicaFetchMinBytes
   private val maxBytes = brokerConfig.replicaFetchResponseMaxBytes
   private val fetchSize = brokerConfig.replicaFetchMaxBytes
-  private val brokerSupportsLeaderEpochRequest: Boolean = brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV2
-
+  private val brokerSupportsLeaderEpochRequest = brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV2
   private val fetchSessionHandler = new FetchSessionHandler(logContext, sourceBroker.id)
 
-  protected def getReplica(tp: TopicPartition): Option[Replica] = {
-    replicaMgr.getReplica(tp)
+  override protected def latestEpoch(topicPartition: TopicPartition): Option[Int] = {
+    replicaMgr.getReplicaOrException(topicPartition).epochs.map(_.latestEpoch)
+  }
+
+  override protected def logEndOffset(topicPartition: TopicPartition): Long = {
+    replicaMgr.getReplicaOrException(topicPartition).logEndOffset.messageOffset
+  }
+
+  override protected def endOffsetForEpoch(topicPartition: TopicPartition, epoch: Int): Option[OffsetAndEpoch] = {
+    val replica = replicaMgr.getReplicaOrException(topicPartition)
+    replica.epochs.flatMap { epochCache =>
+      val (foundEpoch, foundOffset) = epochCache.endOffsetFor(epoch)
+      if (foundOffset == UNDEFINED_EPOCH_OFFSET)
+        None
+      else
+        Some(OffsetAndEpoch(foundOffset, foundEpoch))
+    }
   }
 
   override def initiateShutdown(): Boolean = {
@@ -105,9 +117,12 @@ class ReplicaFetcherThread(name: String,
   }
 
   // process fetched data
-  def processPartitionData(topicPartition: TopicPartition, fetchOffset: Long, partitionData: PD, records: MemoryRecords) {
+  override def processPartitionData(topicPartition: TopicPartition,
+                                    fetchOffset: Long,
+                                    partitionData: PD): Option[LogAppendInfo] = {
     val replica = replicaMgr.getReplicaOrException(topicPartition)
     val partition = replicaMgr.getPartition(topicPartition).get
+    val records = toMemoryRecords(partitionData.records)
 
     maybeWarnIfOversizedRecords(records, topicPartition)
 
@@ -120,7 +135,7 @@ class ReplicaFetcherThread(name: String,
         .format(replica.logEndOffset.messageOffset, topicPartition, records.sizeInBytes, partitionData.highWatermark))
 
     // Append the leader's messages to the log
-    partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = false)
+    val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, isFuture = false)
 
     if (isTraceEnabled)
       trace("Follower has replica log end offset %d after appending %d bytes of messages for partition %s"
@@ -140,6 +155,8 @@ class ReplicaFetcherThread(name: String,
     if (quota.isThrottled(topicPartition))
       quota.record(records.sizeInBytes)
     replicaMgr.brokerTopicStats.updateReplicationBytesIn(records.sizeInBytes)
+
+    logAppendInfo
   }
 
   def maybeWarnIfOversizedRecords(records: MemoryRecords, topicPartition: TopicPartition): Unit = {
@@ -151,79 +168,13 @@ class ReplicaFetcherThread(name: String,
         "equal or larger than your settings for max.message.bytes, both at a broker and topic level.")
   }
 
-  /**
-   * Handle a partition whose offset is out of range and return a new fetch offset.
-   */
-  def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = {
-    val replica = replicaMgr.getReplicaOrException(topicPartition)
-    val partition = replicaMgr.getPartition(topicPartition).get
-
-    /**
-     * Unclean leader election: A follower goes down, in the meanwhile the leader keeps appending messages. The follower comes back up
-     * and before it has completely caught up with the leader's logs, all replicas in the ISR go down. The follower is now uncleanly
-     * elected as the new leader, and it starts appending messages from the client. The old leader comes back up, becomes a follower
-     * and it may discover that the current leader's end offset is behind its own end offset.
-     *
-     * In such a case, truncate the current follower's log to the current leader's end offset and continue fetching.
-     *
-     * There is a potential for a mismatch between the logs of the two replicas here. We don't fix this mismatch as of now.
-     */
-    val leaderEndOffset: Long = earliestOrLatestOffset(topicPartition, ListOffsetRequest.LATEST_TIMESTAMP)
-
-    if (leaderEndOffset < replica.logEndOffset.messageOffset) {
-      // Prior to truncating the follower's log, ensure that doing so is not disallowed by the configuration for unclean leader election.
-      // This situation could only happen if the unclean election configuration for a topic changes while a replica is down. Otherwise,
-      // we should never encounter this situation since a non-ISR leader cannot be elected if disallowed by the broker configuration.
-      val adminZkClient = new AdminZkClient(replicaMgr.zkClient)
-      if (!LogConfig.fromProps(brokerConfig.originals, adminZkClient.fetchEntityConfig(
-        ConfigType.Topic, topicPartition.topic)).uncleanLeaderElectionEnable) {
-        // Log a fatal error and shutdown the broker to ensure that data loss does not occur unexpectedly.
-        fatal(s"Exiting because log truncation is not allowed for partition $topicPartition, current leader's " +
-          s"latest offset $leaderEndOffset is less than replica's latest offset ${replica.logEndOffset.messageOffset}")
-        throw new FatalExitError
-      }
-
-      warn(s"Reset fetch offset for partition $topicPartition from ${replica.logEndOffset.messageOffset} to current " +
-        s"leader's latest offset $leaderEndOffset")
-      partition.truncateTo(leaderEndOffset, isFuture = false)
-      replicaMgr.replicaAlterLogDirsManager.markPartitionsForTruncation(brokerConfig.brokerId, topicPartition, leaderEndOffset)
-      leaderEndOffset
-    } else {
-      /**
-       * If the leader's log end offset is greater than the follower's log end offset, there are two possibilities:
-       * 1. The follower could have been down for a long time and when it starts up, its end offset could be smaller than the leader's
-       * start offset because the leader has deleted old logs (log.logEndOffset < leaderStartOffset).
-       * 2. When unclean leader election occurs, it is possible that the old leader's high watermark is greater than
-       * the new leader's log end offset. So when the old leader truncates its offset to its high watermark and starts
-       * to fetch from the new leader, an OffsetOutOfRangeException will be thrown. After that some more messages are
-       * produced to the new leader. While the old leader is trying to handle the OffsetOutOfRangeException and query
-       * the log end offset of the new leader, the new leader's log end offset becomes higher than the follower's log end offset.
-       *
-       * In the first case, the follower's current log end offset is smaller than the leader's log start offset. So the
-       * follower should truncate all its logs, roll out a new segment and start to fetch from the current leader's log
-       * start offset.
-       * In the second case, the follower should just keep the current log segments and retry the fetch. In the second
-       * case, there will be some inconsistency of data between old and new leader. We are not solving it here.
-       * If users want to have strong consistency guarantees, appropriate configurations needs to be set for both
-       * brokers and producers.
-       *
-       * Putting the two cases together, the follower should fetch from the higher one of its replica log end offset
-       * and the current leader's log start offset.
-       *
-       */
-      val leaderStartOffset: Long = earliestOrLatestOffset(topicPartition, ListOffsetRequest.EARLIEST_TIMESTAMP)
-      warn(s"Reset fetch offset for partition $topicPartition from ${replica.logEndOffset.messageOffset} to current " +
-        s"leader's start offset $leaderStartOffset")
-      val offsetToFetch = Math.max(leaderStartOffset, replica.logEndOffset.messageOffset)
-      // Only truncate log when current leader's log start offset is greater than follower's log end offset.
-      if (leaderStartOffset > replica.logEndOffset.messageOffset) {
-        partition.truncateFullyAndStartAt(leaderStartOffset, isFuture = false)
-      }
-      offsetToFetch
-    }
+  override protected def isUncleanLeaderElectionAllowed(topicPartition: TopicPartition): Boolean = {
+    val adminZkClient = new AdminZkClient(replicaMgr.zkClient)
+    LogConfig.fromProps(brokerConfig.originals, adminZkClient.fetchEntityConfig(
+      ConfigType.Topic, topicPartition.topic)).uncleanLeaderElectionEnable
   }
 
-  protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
+  override protected def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
     try {
       val clientResponse = leaderEndpoint.sendRequest(fetchRequest)
       val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse[Records]]
@@ -239,7 +190,15 @@ class ReplicaFetcherThread(name: String,
     }
   }
 
-  private def earliestOrLatestOffset(topicPartition: TopicPartition, earliestOrLatest: Long): Long = {
+  override protected def fetchEarliestOffsetFromLeader(topicPartition: TopicPartition): Long = {
+    fetchOffsetFromLeader(topicPartition, ListOffsetRequest.EARLIEST_TIMESTAMP)
+  }
+
+  override protected def fetchLatestOffsetFromLeader(topicPartition: TopicPartition): Long = {
+    fetchOffsetFromLeader(topicPartition, ListOffsetRequest.LATEST_TIMESTAMP)
+  }
+
+  private def fetchOffsetFromLeader(topicPartition: TopicPartition, earliestOrLatest: Long): Long = {
     val requestBuilder = if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_1_IV2) {
         val partitions = Map(topicPartition -> (earliestOrLatest: java.lang.Long))
         ListOffsetRequest.Builder.forReplica(listOffsetRequestVersion, replicaId).setTargetTimes(partitions.asJava)
@@ -299,19 +258,23 @@ class ReplicaFetcherThread(name: String,
    * Truncate the log for each partition's epoch based on leader's returned epoch and offset.
    * The logic for finding the truncation offset is implemented in AbstractFetcherThread.getOffsetTruncationState
    */
-  override def truncate(tp: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
+  override def truncate(tp: TopicPartition, offsetTruncationState: OffsetTruncationState): Unit = {
     val replica = replicaMgr.getReplicaOrException(tp)
     val partition = replicaMgr.getPartition(tp).get
+    partition.truncateTo(offsetTruncationState.offset, isFuture = false)
 
-    val offsetTruncationState = getOffsetTruncationState(tp, epochEndOffset, replica)
     if (offsetTruncationState.offset < replica.highWatermark.messageOffset)
-      warn(s"Truncating $tp to offset ${offsetTruncationState.offset} below high watermark ${replica.highWatermark.messageOffset}")
-    partition.truncateTo(offsetTruncationState.offset, isFuture = false)
+      warn(s"Truncating $tp to offset ${offsetTruncationState.offset} below high watermark " +
+        s"${replica.highWatermark.messageOffset}")
 
     // mark the future replica for truncation only when we do last truncation
     if (offsetTruncationState.truncationCompleted)
       replicaMgr.replicaAlterLogDirsManager.markPartitionsForTruncation(brokerConfig.brokerId, tp, offsetTruncationState.offset)
-    offsetTruncationState
+  }
+
+  override protected def truncateFullyAndStartAt(topicPartition: TopicPartition, offset: Long): Unit = {
+    val partition = replicaMgr.getPartition(topicPartition).get
+    partition.truncateFullyAndStartAt(offset, isFuture = false)
   }
 
   override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
@@ -360,4 +323,5 @@ class ReplicaFetcherThread(name: String,
     val isReplicaInSync = fetcherLagStats.isReplicaInSync(topicPartition)
     quota.isThrottled(topicPartition) && quota.isQuotaExceeded && !isReplicaInSync
   }
+
 }
diff --git a/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala b/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala
index 6fcf0ccd54b13..392c912b25aff 100644
--- a/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala
+++ b/core/src/test/scala/integration/kafka/server/ReplicaFetcherThreadFatalErrorTest.scala
@@ -88,7 +88,7 @@ class ReplicaFetcherThreadFatalErrorTest extends ZooKeeperTestHarness {
       import params._
       new ReplicaFetcherThread(threadName, fetcherId, sourceBroker, config, replicaManager, metrics, time, quotaManager) {
         override def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = throw new FatalExitError
-        override protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
+        override protected def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
           fetchRequest.fetchData.asScala.keys.toSeq.map { tp =>
             (tp, new FetchResponse.PartitionData[Records](Errors.OFFSET_OUT_OF_RANGE,
               FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET,
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
index 15abc6896b1d9..c456433b6e359 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
@@ -17,20 +17,26 @@
 
 package kafka.server
 
+import java.nio.ByteBuffer
+
 import AbstractFetcherThread._
 import com.yammer.metrics.Metrics
-import kafka.cluster.{BrokerEndPoint, Replica}
+import kafka.cluster.BrokerEndPoint
+import kafka.log.LogAppendInfo
+import kafka.message.NoCompressionCodec
 import kafka.utils.TestUtils
 import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.internals.FatalExitError
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
-import org.apache.kafka.common.record.{CompressionType, MemoryRecords, Records, SimpleRecord}
+import org.apache.kafka.common.record._
 import org.apache.kafka.common.requests.{EpochEndOffset, FetchRequest}
-import org.apache.kafka.common.requests.FetchResponse.PartitionData
-import org.junit.Assert.{assertFalse, assertTrue}
+import org.apache.kafka.common.utils.Time
+import org.junit.Assert._
 import org.junit.{Before, Test}
 
 import scala.collection.JavaConverters._
 import scala.collection.{Map, Set, mutable}
+import scala.util.Random
 
 class AbstractFetcherThreadTest {
 
@@ -40,174 +46,412 @@ class AbstractFetcherThreadTest {
       Metrics.defaultRegistry().removeMetric(metricName)
   }
 
+  private def allMetricsNames: Set[String] = Metrics.defaultRegistry().allMetrics().asScala.keySet.map(_.getName)
+
+  private def mkBatch(baseOffset: Long, leaderEpoch: Int, records: SimpleRecord*): RecordBatch = {
+    MemoryRecords.withRecords(baseOffset, CompressionType.NONE, leaderEpoch, records: _*)
+      .batches.asScala.head
+  }
+
   @Test
-  def testMetricsRemovedOnShutdown() {
+  def testMetricsRemovedOnShutdown(): Unit = {
     val partition = new TopicPartition("topic", 0)
-    val fetcherThread = new DummyFetcherThread("dummy", "client", new BrokerEndPoint(0, "localhost", 9092))
-
-    fetcherThread.start()
+    val fetcher = new MockFetcherThread
 
     // add one partition to create the consumer lag metric
-    fetcherThread.addPartitions(Map(partition -> 0L))
+    fetcher.setReplicaState(partition, MockFetcherThread.PartitionState())
+    fetcher.addPartitions(Map(partition -> 0L))
+    fetcher.setLeaderState(partition, MockFetcherThread.PartitionState())
+
+    fetcher.start()
 
     // wait until all fetcher metrics are present
     TestUtils.waitUntilTrue(() =>
       allMetricsNames == Set(FetcherMetrics.BytesPerSec, FetcherMetrics.RequestsPerSec, FetcherMetrics.ConsumerLag),
       "Failed waiting for all fetcher metrics to be registered")
 
-    fetcherThread.shutdown()
+    fetcher.shutdown()
 
     // after shutdown, they should be gone
     assertTrue(Metrics.defaultRegistry().allMetrics().isEmpty)
   }
 
   @Test
-  def testConsumerLagRemovedWithPartition() {
+  def testConsumerLagRemovedWithPartition(): Unit = {
     val partition = new TopicPartition("topic", 0)
-    val fetcherThread = new DummyFetcherThread("dummy", "client", new BrokerEndPoint(0, "localhost", 9092))
-
-    fetcherThread.start()
+    val fetcher = new MockFetcherThread
 
     // add one partition to create the consumer lag metric
-    fetcherThread.addPartitions(Map(partition -> 0L))
+    fetcher.setReplicaState(partition, MockFetcherThread.PartitionState())
+    fetcher.addPartitions(Map(partition -> 0L))
+    fetcher.setLeaderState(partition, MockFetcherThread.PartitionState())
+
+    fetcher.doWork()
 
-    // wait until lag metric is present
-    TestUtils.waitUntilTrue(() => allMetricsNames(FetcherMetrics.ConsumerLag),
-      "Failed waiting for consumer lag metric")
+    assertTrue("Failed waiting for consumer lag metric",
+      allMetricsNames(FetcherMetrics.ConsumerLag))
 
     // remove the partition to simulate leader migration
-    fetcherThread.removePartitions(Set(partition))
+    fetcher.removePartitions(Set(partition))
 
     // the lag metric should now be gone
     assertFalse(allMetricsNames(FetcherMetrics.ConsumerLag))
+  }
+
+  @Test
+  def testSimpleFetch(): Unit = {
+    val partition = new TopicPartition("topic", 0)
+    val fetcher = new MockFetcherThread
+
+    fetcher.setReplicaState(partition, MockFetcherThread.PartitionState())
+    fetcher.addPartitions(Map(partition -> 0L))
+
+    val batch = mkBatch(baseOffset = 0L, leaderEpoch = 0,
+      new SimpleRecord("a".getBytes), new SimpleRecord("b".getBytes))
+    val leaderState = MockFetcherThread.PartitionState(Seq(batch), highWatermark = 2L)
+    fetcher.setLeaderState(partition, leaderState)
+
+    fetcher.doWork()
+
+    val replicaState = fetcher.replicaPartitionState(partition)
+    assertEquals(2L, replicaState.logEndOffset)
+    assertEquals(2L, replicaState.highWatermark)
+  }
+
+  @Test
+  def testTruncation(): Unit = {
+    val partition = new TopicPartition("topic", 0)
+    val fetcher = new MockFetcherThread
+
+    val replicaLog = Seq(
+      mkBatch(baseOffset = 0, leaderEpoch = 0, new SimpleRecord("a".getBytes)),
+      mkBatch(baseOffset = 1, leaderEpoch = 2, new SimpleRecord("b".getBytes)),
+      mkBatch(baseOffset = 2, leaderEpoch = 4, new SimpleRecord("c".getBytes)))
+
+    val replicaState = MockFetcherThread.PartitionState(replicaLog, highWatermark = 0L)
+    fetcher.setReplicaState(partition, replicaState)
+    fetcher.addPartitions(Map(partition -> 3L))
+
+    val leaderLog = Seq(
+      mkBatch(baseOffset = 0, leaderEpoch = 1, new SimpleRecord("a".getBytes)),
+      mkBatch(baseOffset = 1, leaderEpoch = 3, new SimpleRecord("b".getBytes)),
+      mkBatch(baseOffset = 2, leaderEpoch = 5, new SimpleRecord("c".getBytes)))
+
+    val leaderState = MockFetcherThread.PartitionState(leaderLog, highWatermark = 2L)
+    fetcher.setLeaderState(partition, leaderState)
 
-    fetcherThread.shutdown()
+    TestUtils.waitUntilTrue(() => {
+      fetcher.doWork()
+      fetcher.replicaPartitionState(partition).log == fetcher.leaderPartitionState(partition).log
+    }, "Failed to reconcile leader and follower logs")
+
+    assertEquals(leaderState.logStartOffset, replicaState.logStartOffset)
+    assertEquals(leaderState.logEndOffset, replicaState.logEndOffset)
+    assertEquals(leaderState.highWatermark, replicaState.highWatermark)
   }
 
-  private def allMetricsNames = Metrics.defaultRegistry().allMetrics().asScala.keySet.map(_.getName)
+  @Test(expected = classOf[FatalExitError])
+  def testFollowerFetchOutOfRangeHighUncleanLeaderElectionDisallowed(): Unit = {
+    val partition = new TopicPartition("topic", 0)
+    val fetcher = new MockFetcherThread(isUncleanLeaderElectionAllowed = false)
+
+    val replicaLog = Seq(
+      mkBatch(baseOffset = 0, leaderEpoch = 0, new SimpleRecord("a".getBytes)),
+      mkBatch(baseOffset = 1, leaderEpoch = 2, new SimpleRecord("b".getBytes)),
+      mkBatch(baseOffset = 2, leaderEpoch = 4, new SimpleRecord("c".getBytes)))
+
+    val replicaState = MockFetcherThread.PartitionState(replicaLog, highWatermark = 0L)
+    fetcher.setReplicaState(partition, replicaState)
+    fetcher.addPartitions(Map(partition -> 3L))
+
+    val leaderLog = Seq(
+      mkBatch(baseOffset = 0, leaderEpoch = 0, new SimpleRecord("a".getBytes)),
+      mkBatch(baseOffset = 1, leaderEpoch = 2, new SimpleRecord("b".getBytes)),
+      mkBatch(baseOffset = 2, leaderEpoch = 4, new SimpleRecord("c".getBytes)))
 
-  protected def fetchRequestBuilder(partitionMap: collection.Map[TopicPartition, PartitionFetchState]): FetchRequest.Builder = {
-    val partitionData = partitionMap.map { case (tp, fetchState) =>
-      tp -> new FetchRequest.PartitionData(fetchState.fetchOffset, 0, 1024 * 1024)
-    }.toMap.asJava
-    FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion, 0, 0, 1, partitionData)
+    val leaderState = MockFetcherThread.PartitionState(leaderLog, highWatermark = 2L)
+    fetcher.setLeaderState(partition, leaderState)
+
+    // initial truncation and verify that the log end offset is updated
+    fetcher.doWork()
+    assertEquals(3L, replicaState.logEndOffset)
+    assertFalse(fetcher.partitionStates.stateValue(partition).isTruncatingLog)
+
+    // To hit this case, we have to change the leader log without going through the truncation phase
+    leaderState.log.clear()
+    leaderState.logEndOffset = 0L
+    leaderState.logStartOffset = 0L
+    leaderState.highWatermark = 0L
+
+    fetcher.doWork()
   }
 
-  class DummyFetcherThread(name: String,
-                           clientId: String,
-                           sourceBroker: BrokerEndPoint,
-                           fetchBackOffMs: Int = 0)
-    extends AbstractFetcherThread(name, clientId, sourceBroker,
-      fetchBackOffMs,
-      isInterruptible = true,
-      includeLogTruncation = false) {
+  @Test
+  def testFollowerFetchOutOfRangeLow(): Unit = {
+    val partition = new TopicPartition("topic", 0)
+    val fetcher = new MockFetcherThread
 
-    protected def getReplica(tp: TopicPartition): Option[Replica] = None
+    // The follower begins from an offset which is behind the leader's log start offset
+    val replicaLog = Seq(
+      mkBatch(baseOffset = 0, leaderEpoch = 0, new SimpleRecord("a".getBytes)))
 
-    override def processPartitionData(topicPartition: TopicPartition,
-                                      fetchOffset: Long,
-                                      partitionData: PD,
-                                      records: MemoryRecords): Unit = {}
+    val replicaState = MockFetcherThread.PartitionState(replicaLog, highWatermark = 0L)
+    fetcher.setReplicaState(partition, replicaState)
+    fetcher.addPartitions(Map(partition -> 3L))
 
-    override def handleOffsetOutOfRange(topicPartition: TopicPartition): Long = 0L
+    val leaderLog = Seq(
+      mkBatch(baseOffset = 2, leaderEpoch = 4, new SimpleRecord("c".getBytes)))
 
-    override protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] =
-      fetchRequest.fetchData.asScala.mapValues(_ => new PartitionData[Records](Errors.NONE, 0, 0, 0,
-        Seq.empty.asJava, MemoryRecords.EMPTY)).toSeq
+    val leaderState = MockFetcherThread.PartitionState(leaderLog, highWatermark = 2L)
+    fetcher.setLeaderState(partition, leaderState)
 
-    override protected def buildFetch(partitionMap: collection.Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
-      ResultWithPartitions(Some(fetchRequestBuilder(partitionMap)), Set())
-    }
+    // initial truncation and verify that the log start offset is updated
+    fetcher.doWork()
+    assertFalse(fetcher.partitionStates.stateValue(partition).isTruncatingLog)
+    assertEquals(2, replicaState.logStartOffset)
+    assertEquals(List(), replicaState.log.toList)
 
-    override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = { Map() }
+    TestUtils.waitUntilTrue(() => {
+      fetcher.doWork()
+      fetcher.replicaPartitionState(partition).log == fetcher.leaderPartitionState(partition).log
+    }, "Failed to reconcile leader and follower logs")
 
-    override def truncate(tp: TopicPartition, epochEndOffset: EpochEndOffset): OffsetTruncationState = {
-      OffsetTruncationState(epochEndOffset.endOffset, truncationCompleted = true)
-    }
+    assertEquals(leaderState.logStartOffset, replicaState.logStartOffset)
+    assertEquals(leaderState.logEndOffset, replicaState.logEndOffset)
+    assertEquals(leaderState.highWatermark, replicaState.highWatermark)
   }
 
   @Test
-  def testFetchRequestCorruptedMessageException() {
+  def testCorruptMessage(): Unit = {
     val partition = new TopicPartition("topic", 0)
-    val fetcherThread = new CorruptingFetcherThread("test", "client", new BrokerEndPoint(0, "localhost", 9092),
-      fetchBackOffMs = 1)
 
-    fetcherThread.start()
+    val fetcher = new MockFetcherThread {
+      var fetchedOnce = false
+      override def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
+        val fetchedData = super.fetchFromLeader(fetchRequest)
+        if (!fetchedOnce) {
+          val records = fetchedData.head._2.records.asInstanceOf[MemoryRecords]
+          val buffer = records.buffer()
+          buffer.putInt(15, buffer.getInt(15) ^ 23422)
+          buffer.putInt(30, buffer.getInt(30) ^ 93242)
+          fetchedOnce = true
+        }
+        fetchedData
+      }
+    }
+
+    fetcher.setReplicaState(partition, MockFetcherThread.PartitionState())
+    fetcher.addPartitions(Map(partition -> 0L))
 
-    // Add one partition for fetching
-    fetcherThread.addPartitions(Map(partition -> 0L))
+    val batch = mkBatch(baseOffset = 0L, leaderEpoch = 0,
+      new SimpleRecord("a".getBytes), new SimpleRecord("b".getBytes))
+    val leaderState = MockFetcherThread.PartitionState(Seq(batch), highWatermark = 2L)
+    fetcher.setLeaderState(partition, leaderState)
 
-    // Wait until fetcherThread finishes the work
-    TestUtils.waitUntilTrue(() => fetcherThread.fetchCount > 3, "Failed waiting for fetcherThread to finish the work")
+    fetcher.doWork() // fails with corrupt record
+    fetcher.doWork() // should succeed
+
+    val replicaState = fetcher.replicaPartitionState(partition)
+    assertEquals(2L, replicaState.logEndOffset)
+  }
 
-    fetcherThread.shutdown()
+  object MockFetcherThread {
+    class PartitionState(var log: mutable.Buffer[RecordBatch],
+                         var logStartOffset: Long,
+                         var logEndOffset: Long,
+                         var highWatermark: Long)
+
+    object PartitionState {
+      def apply(log: Seq[RecordBatch], highWatermark: Long): PartitionState = {
+        val logStartOffset = log.headOption.map(_.baseOffset).getOrElse(0L)
+        val logEndOffset = log.lastOption.map(_.nextOffset).getOrElse(0L)
+        new PartitionState(log.toBuffer, logStartOffset, logEndOffset, highWatermark)
+      }
 
-    // The fetcherThread should have fetched two normal messages
-    assertTrue(fetcherThread.logEndOffset == 2)
+      def apply(): PartitionState = {
+        apply(Seq(), 0L)
+      }
+    }
   }
 
-  class CorruptingFetcherThread(name: String,
-                                clientId: String,
-                                sourceBroker: BrokerEndPoint,
-                                fetchBackOffMs: Int = 0)
-    extends DummyFetcherThread(name, clientId, sourceBroker, fetchBackOffMs) {
+  class MockFetcherThread(val replicaId: Int = 0,
+                          val leaderId: Int = 1,
+                          isUncleanLeaderElectionAllowed: Boolean = true)
+    extends AbstractFetcherThread("mock-fetcher",
+      clientId = "mock-fetcher",
+      sourceBroker = new BrokerEndPoint(leaderId, host = "localhost", port = Random.nextInt())) {
 
-    @volatile var logEndOffset = 0L
-    @volatile var fetchCount = 0
+    import MockFetcherThread.PartitionState
 
-    private val normalPartitionDataSet = List[PartitionData[Records]](
-      new PartitionData(Errors.NONE, 0L, 0L, 0L, Seq.empty.asJava,
-        MemoryRecords.withRecords(0L, CompressionType.NONE, new SimpleRecord("hello".getBytes))),
-      new PartitionData(Errors.NONE, 0L, 0L, 0L, Seq.empty.asJava,
-        MemoryRecords.withRecords(1L, CompressionType.NONE, new SimpleRecord("hello".getBytes)))
-    )
+    private val replicaPartitionStates = mutable.Map[TopicPartition, PartitionState]()
+    private val leaderPartitionStates = mutable.Map[TopicPartition, PartitionState]()
+
+    def setLeaderState(topicPartition: TopicPartition, state: PartitionState): Unit = {
+      leaderPartitionStates.put(topicPartition, state)
+    }
+
+    def setReplicaState(topicPartition: TopicPartition, state: PartitionState): Unit = {
+      replicaPartitionStates.put(topicPartition, state)
+    }
+
+    def replicaPartitionState(topicPartition: TopicPartition): PartitionState = {
+      replicaPartitionStates.getOrElse(topicPartition,
+        throw new IllegalArgumentException(s"Unknown partition $topicPartition"))
+    }
+
+    def leaderPartitionState(topicPartition: TopicPartition): PartitionState = {
+      leaderPartitionStates.getOrElse(topicPartition,
+        throw new IllegalArgumentException(s"Unknown partition $topicPartition"))
+    }
 
     override def processPartitionData(topicPartition: TopicPartition,
                                       fetchOffset: Long,
-                                      partitionData: PD,
-                                      records: MemoryRecords): Unit = {
+                                      partitionData: PD): Option[LogAppendInfo] = {
+      val state = replicaPartitionState(topicPartition)
+
       // Throw exception if the fetchOffset does not match the fetcherThread partition state
-      if (fetchOffset != logEndOffset)
-        throw new RuntimeException(
-          "Offset mismatch for partition %s: fetched offset = %d, log end offset = %d."
-            .format(topicPartition, fetchOffset, logEndOffset))
+      if (fetchOffset != state.logEndOffset)
+        throw new RuntimeException(s"Offset mismatch for partition $topicPartition: " +
+          s"fetched offset = $fetchOffset, log end offset = ${state.logEndOffset}.")
 
       // Now check message's crc
-      for (batch <- records.batches.asScala) {
+      val batches = partitionData.records.batches.asScala
+      var maxTimestamp = RecordBatch.NO_TIMESTAMP
+      var offsetOfMaxTimestamp = -1L
+      var lastOffset = state.logEndOffset
+
+      for (batch <- batches) {
         batch.ensureValid()
-        logEndOffset = batch.nextOffset
+        if (batch.maxTimestamp > maxTimestamp) {
+          maxTimestamp = batch.maxTimestamp
+          offsetOfMaxTimestamp = batch.baseOffset
+        }
+        state.log.append(batch)
+        state.logEndOffset = batch.nextOffset
+        lastOffset = batch.lastOffset
+      }
+
+      state.logStartOffset = partitionData.logStartOffset
+      state.highWatermark = partitionData.highWatermark
+
+      Some(LogAppendInfo(firstOffset = Some(fetchOffset),
+        lastOffset = lastOffset,
+        maxTimestamp = maxTimestamp,
+        offsetOfMaxTimestamp = offsetOfMaxTimestamp,
+        logAppendTime = Time.SYSTEM.milliseconds(),
+        logStartOffset = state.logStartOffset,
+        recordConversionStats = RecordConversionStats.EMPTY,
+        sourceCodec = NoCompressionCodec,
+        targetCodec = NoCompressionCodec,
+        shallowCount = batches.size,
+        validBytes = partitionData.records.sizeInBytes,
+        offsetsMonotonic = true,
+        lastOffsetOfFirstBatch = batches.headOption.map(_.lastOffset).getOrElse(-1)))
+    }
+
+    override def truncate(topicPartition: TopicPartition, truncationState: OffsetTruncationState): Unit = {
+     val state = replicaPartitionState(topicPartition)
+     state.log = state.log.takeWhile { batch =>
+       batch.lastOffset < truncationState.offset
+     }
+     state.logEndOffset = state.log.lastOption.map(_.lastOffset + 1).getOrElse(state.logStartOffset)
+     state.highWatermark = math.min(state.highWatermark, state.logEndOffset)
+    }
+
+    override def truncateFullyAndStartAt(topicPartition: TopicPartition, offset: Long): Unit = {
+      val state = replicaPartitionState(topicPartition)
+      state.log.clear()
+      state.logStartOffset = offset
+      state.logEndOffset = offset
+      state.highWatermark = offset
+    }
+
+    override def buildFetch(partitionMap: Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
+      val fetchData = mutable.Map.empty[TopicPartition, FetchRequest.PartitionData]
+      partitionMap.foreach { case (partition, state) =>
+        if (state.isReadyForFetch) {
+          val replicaState = replicaPartitionState(partition)
+          fetchData.put(partition, new FetchRequest.PartitionData(state.fetchOffset, replicaState.logStartOffset, 1024 * 1024))
+        }
       }
+      val fetchRequest = FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion, replicaId, 0, 1, fetchData.asJava)
+      ResultWithPartitions(Some(fetchRequest), Set.empty)
     }
 
-    override protected def fetch(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
-      fetchCount += 1
-      // Set the first fetch to get a corrupted message
-      if (fetchCount == 1) {
-        val record = new SimpleRecord("hello".getBytes())
-        val records = MemoryRecords.withRecords(CompressionType.NONE, record)
-        val buffer = records.buffer
-
-        // flip some bits in the message to ensure the crc fails
-        buffer.putInt(15, buffer.getInt(15) ^ 23422)
-        buffer.putInt(30, buffer.getInt(30) ^ 93242)
-        fetchRequest.fetchData.asScala.mapValues(_ => new PartitionData[Records](Errors.NONE, 0L, 0L, 0L,
-          Seq.empty.asJava, records)).toSeq
-      } else {
-        // Then, the following fetches get the normal data
-        fetchRequest.fetchData.asScala.mapValues(v => normalPartitionDataSet(v.fetchOffset.toInt)).toSeq
+    override def isUncleanLeaderElectionAllowed(topicPartition: TopicPartition): Boolean = {
+      isUncleanLeaderElectionAllowed
+    }
+
+    override def latestEpoch(topicPartition: TopicPartition): Option[Int] = {
+      val state = replicaPartitionState(topicPartition)
+      state.log.lastOption.map(_.partitionLeaderEpoch).orElse(Some(EpochEndOffset.UNDEFINED_EPOCH))
+    }
+
+    override def logEndOffset(topicPartition: TopicPartition): Long = replicaPartitionState(topicPartition).logEndOffset
+
+    override def endOffsetForEpoch(topicPartition: TopicPartition, epoch: Int): Option[OffsetAndEpoch] = {
+      lookupEndOffsetForEpoch(epoch, replicaPartitionState(topicPartition))
+    }
+
+    private def lookupEndOffsetForEpoch(epoch: Int, partitionState: PartitionState): Option[OffsetAndEpoch] = {
+      var epochLowerBound = EpochEndOffset.UNDEFINED_EPOCH
+      for (batch <- partitionState.log) {
+        if (batch.partitionLeaderEpoch > epoch) {
+          return Some(OffsetAndEpoch(batch.baseOffset, epochLowerBound))
+        }
+        epochLowerBound = batch.partitionLeaderEpoch
       }
+      None
     }
 
-    override protected def buildFetch(partitionMap: collection.Map[TopicPartition, PartitionFetchState]): ResultWithPartitions[Option[FetchRequest.Builder]] = {
-      val requestMap = new mutable.HashMap[TopicPartition, Long]
-      partitionMap.foreach { case (topicPartition, partitionFetchState) =>
-        // Add backoff delay check
-        if (partitionFetchState.isReadyForFetch)
-          requestMap.put(topicPartition, partitionFetchState.fetchOffset)
+    override def fetchEpochsFromLeader(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
+      val endOffsets = mutable.Map[TopicPartition, EpochEndOffset]()
+      partitions.foreach { case (partition, epoch) =>
+        val state = leaderPartitionState(partition)
+        val epochEndOffset = lookupEndOffsetForEpoch(epoch, state) match {
+          case Some(OffsetAndEpoch(offset, epoch)) =>
+            new EpochEndOffset(Errors.NONE, epoch, offset)
+          case None =>
+            new EpochEndOffset(Errors.NONE, EpochEndOffset.UNDEFINED_EPOCH, EpochEndOffset.UNDEFINED_EPOCH_OFFSET)
+        }
+        endOffsets.put(partition, epochEndOffset)
       }
-      ResultWithPartitions(Some(fetchRequestBuilder(partitionMap)), Set())
+      endOffsets
     }
 
+    override def fetchFromLeader(fetchRequest: FetchRequest.Builder): Seq[(TopicPartition, PD)] = {
+      fetchRequest.fetchData.asScala.map { case (partition, fetchData) =>
+        val state = leaderPartitionState(partition)
+        val (error, records) = if (fetchData.fetchOffset > state.logEndOffset || fetchData.fetchOffset < state.logStartOffset) {
+          (Errors.OFFSET_OUT_OF_RANGE, MemoryRecords.EMPTY)
+        } else {
+          // for simplicity, we fetch only one batch at a time
+          val records = state.log.find(_.baseOffset >= fetchData.fetchOffset) match {
+            case Some(batch) =>
+              val buffer = ByteBuffer.allocate(batch.sizeInBytes())
+              batch.writeTo(buffer)
+              buffer.flip()
+              MemoryRecords.readableRecords(buffer)
+
+            case None =>
+              MemoryRecords.EMPTY
+          }
+
+          (Errors.NONE, records)
+        }
+
+        (partition, new PD(error, state.highWatermark, state.highWatermark, state.logStartOffset,
+          List.empty.asJava, records))
+      }.toSeq
+    }
+
+    override protected def fetchEarliestOffsetFromLeader(topicPartition: TopicPartition): Long = {
+      leaderPartitionState(topicPartition).logStartOffset
+    }
+
+    override protected def fetchLatestOffsetFromLeader(topicPartition: TopicPartition): Long = {
+      leaderPartitionState(topicPartition).logEndOffset
+    }
   }
 
 }

From 958cdca9bece8b65ceb204e1c7a14cf44729bb66 Mon Sep 17 00:00:00 2001
From: Dhruvil Shah 
Date: Sat, 8 Sep 2018 18:01:01 -0700
Subject: [PATCH 0786/1847] KAFKA-7385; Fix log cleaner behavior when only
 empty batches are retained (#5623)

With idempotent/transactional producers, we may leave empty batches in the log during log compaction. When filtering the data, we keep track of state like `maxOffset` and `maxTimestamp` of filtered data. This patch ensures we maintain this state correctly for the case when only empty batches are left in `MemoryRecords#filterTo`. Without this patch, we did not initialize `maxOffset` in this edge case which led us to append data to the log with `maxOffset` = -1L, causing the append to fail and log cleaner to crash.

Reviewers: Jason Gustafson 
---
 .../kafka/common/record/MemoryRecords.java    | 143 +++++++++-------
 .../consumer/internals/FetcherTest.java       |   4 +-
 .../common/record/MemoryRecordsTest.java      | 153 +++++++++++++-----
 .../src/main/scala/kafka/log/LogCleaner.scala |   2 +-
 .../scala/unit/kafka/log/LogCleanerTest.scala |  45 ++++--
 5 files changed, 229 insertions(+), 118 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
index 55a471149c6ca..af62e09c57ecd 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
@@ -160,20 +160,14 @@ public FilterResult filterTo(TopicPartition partition, RecordFilter filter, Byte
     private static FilterResult filterTo(TopicPartition partition, Iterable batches,
                                          RecordFilter filter, ByteBuffer destinationBuffer, int maxRecordBatchSize,
                                          BufferSupplier decompressionBufferSupplier) {
-        long maxTimestamp = RecordBatch.NO_TIMESTAMP;
-        long maxOffset = -1L;
-        long shallowOffsetOfMaxTimestamp = -1L;
-        int messagesRead = 0;
-        int bytesRead = 0; // bytes processed from `batches`
-        int messagesRetained = 0;
-        int bytesRetained = 0;
-
+        FilterResult filterResult = new FilterResult(destinationBuffer);
         ByteBufferOutputStream bufferOutputStream = new ByteBufferOutputStream(destinationBuffer);
 
         for (MutableRecordBatch batch : batches) {
-            bytesRead += batch.sizeInBytes();
-
+            long maxOffset = -1L;
             BatchRetention batchRetention = filter.checkBatchRetention(batch);
+            filterResult.bytesRead += batch.sizeInBytes();
+
             if (batchRetention == BatchRetention.DELETE)
                 continue;
 
@@ -189,7 +183,7 @@ private static FilterResult filterTo(TopicPartition partition, Iterable iterator = batch.streamingIterator(decompressionBufferSupplier)) {
                 while (iterator.hasNext()) {
                     Record record = iterator.next();
-                    messagesRead += 1;
+                    filterResult.messagesRead += 1;
 
                     if (filter.shouldRetainRecord(batch, record)) {
                         // Check for log corruption due to KAFKA-4298. If we find it, make sure that we overwrite
@@ -210,20 +204,11 @@ private static FilterResult filterTo(TopicPartition partition, Iterable maxTimestamp) {
-                        maxTimestamp = batch.maxTimestamp();
-                        shallowOffsetOfMaxTimestamp = batch.lastOffset();
-                    }
+                    filterResult.updateRetainedBatchMetadata(batch, retainedRecords.size(), false);
                 } else {
                     MemoryRecordsBuilder builder = buildRetainedRecordsInto(batch, retainedRecords, bufferOutputStream);
                     MemoryRecords records = builder.build();
                     int filteredBatchSize = records.sizeInBytes();
-
-                    messagesRetained += retainedRecords.size();
-                    bytesRetained += filteredBatchSize;
-
                     if (filteredBatchSize > batch.sizeInBytes() && filteredBatchSize > maxRecordBatchSize)
                         log.warn("Record batch from {} with last offset {} exceeded max record batch size {} after cleaning " +
                                         "(new size is {}). Consumers with version earlier than 0.10.1.0 may need to " +
@@ -231,10 +216,8 @@ private static FilterResult filterTo(TopicPartition partition, Iterable maxTimestamp) {
-                        maxTimestamp = info.maxTimestamp;
-                        shallowOffsetOfMaxTimestamp = info.shallowOffsetOfMaxTimestamp;
-                    }
+                    filterResult.updateRetainedBatchMetadata(info.maxTimestamp, info.shallowOffsetOfMaxTimestamp,
+                            maxOffset, retainedRecords.size(), filteredBatchSize);
                 }
             } else if (batchRetention == BatchRetention.RETAIN_EMPTY) {
                 if (batchMagic < RecordBatch.MAGIC_VALUE_V2)
@@ -245,18 +228,19 @@ private static FilterResult filterTo(TopicPartition partition, Iterable this.maxTimestamp) {
+                this.maxTimestamp = maxTimestamp;
+                this.shallowOffsetOfMaxTimestamp = shallowOffsetOfMaxTimestamp;
+            }
+            this.maxOffset = Math.max(maxOffset, this.maxOffset);
+            this.messagesRetained += messagesRetained;
+            this.bytesRetained += bytesRetained;
+        }
+
+        private void validateBatchMetadata(long maxTimestamp, long shallowOffsetOfMaxTimestamp, long maxOffset) {
+            if (maxTimestamp != RecordBatch.NO_TIMESTAMP && shallowOffsetOfMaxTimestamp < 0)
+                throw new IllegalArgumentException("shallowOffset undefined for maximum timestamp " + maxTimestamp);
+            if (maxOffset < 0)
+                throw new IllegalArgumentException("maxOffset undefined");
+        }
+
+        public ByteBuffer outputBuffer() {
+            return outputBuffer;
+        }
+
+        public int messagesRead() {
+            return messagesRead;
+        }
+
+        public int bytesRead() {
+            return bytesRead;
+        }
+
+        public int messagesRetained() {
+            return messagesRetained;
+        }
+
+        public int bytesRetained() {
+            return bytesRetained;
+        }
+
+        public long maxOffset() {
+            return maxOffset;
+        }
+
+        public long maxTimestamp() {
+            return maxTimestamp;
+        }
+
+        public long shallowOffsetOfMaxTimestamp() {
+            return shallowOffsetOfMaxTimestamp;
         }
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 1a82faa0a3dd9..fd550d61da515 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -2113,8 +2113,8 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
                 return record.key() != null;
             }
         }, ByteBuffer.allocate(1024), Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
-        result.output.flip();
-        MemoryRecords compactedRecords = MemoryRecords.readableRecords(result.output);
+        result.outputBuffer().flip();
+        MemoryRecords compactedRecords = MemoryRecords.readableRecords(result.outputBuffer());
 
         subscriptions.assignFromUser(singleton(tp0));
         subscriptions.seek(tp0, 0);
diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
index 61d8a00865bc5..579fb74b44a83 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java
@@ -252,6 +252,7 @@ public void testFilterToEmptyBatchRetention() {
                 long baseOffset = 3L;
                 int baseSequence = 10;
                 int partitionLeaderEpoch = 293;
+                int numRecords = 2;
 
                 MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME,
                         baseOffset, RecordBatch.NO_TIMESTAMP, producerId, producerEpoch, baseSequence, isTransactional,
@@ -259,22 +260,34 @@ public void testFilterToEmptyBatchRetention() {
                 builder.append(11L, "2".getBytes(), "b".getBytes());
                 builder.append(12L, "3".getBytes(), "c".getBytes());
                 builder.close();
+                MemoryRecords records = builder.build();
 
                 ByteBuffer filtered = ByteBuffer.allocate(2048);
-                builder.build().filterTo(new TopicPartition("foo", 0), new MemoryRecords.RecordFilter() {
-                    @Override
-                    protected BatchRetention checkBatchRetention(RecordBatch batch) {
-                        // retain all batches
-                        return BatchRetention.RETAIN_EMPTY;
-                    }
-
-                    @Override
-                    protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
-                        // delete the records
-                        return false;
-                    }
-                }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
-
+                MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0),
+                        new MemoryRecords.RecordFilter() {
+                            @Override
+                            protected BatchRetention checkBatchRetention(RecordBatch batch) {
+                                // retain all batches
+                                return BatchRetention.RETAIN_EMPTY;
+                            }
+
+                            @Override
+                            protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
+                                // delete the records
+                                return false;
+                            }
+                        }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
+
+                // Verify filter result
+                assertEquals(numRecords, filterResult.messagesRead());
+                assertEquals(records.sizeInBytes(), filterResult.bytesRead());
+                assertEquals(baseOffset + 1, filterResult.maxOffset());
+                assertEquals(0, filterResult.messagesRetained());
+                assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filterResult.bytesRetained());
+                assertEquals(12, filterResult.maxTimestamp());
+                assertEquals(baseOffset + 1, filterResult.shallowOffsetOfMaxTimestamp());
+
+                // Verify filtered records
                 filtered.flip();
                 MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
 
@@ -294,6 +307,55 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
         }
     }
 
+    @Test
+    public void testEmptyBatchRetention() {
+        if (magic >= RecordBatch.MAGIC_VALUE_V2) {
+            ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD);
+            long producerId = 23L;
+            short producerEpoch = 5;
+            long baseOffset = 3L;
+            int baseSequence = 10;
+            int partitionLeaderEpoch = 293;
+            long timestamp = System.currentTimeMillis();
+
+            DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, producerId, producerEpoch,
+                    baseSequence, baseOffset, baseOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME,
+                    timestamp, false, false);
+            buffer.flip();
+
+            ByteBuffer filtered = ByteBuffer.allocate(2048);
+            MemoryRecords records = MemoryRecords.readableRecords(buffer);
+            MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0),
+                    new MemoryRecords.RecordFilter() {
+                        @Override
+                        protected BatchRetention checkBatchRetention(RecordBatch batch) {
+                            // retain all batches
+                            return BatchRetention.RETAIN_EMPTY;
+                        }
+
+                        @Override
+                        protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
+                            return false;
+                        }
+                    }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
+
+            // Verify filter result
+            assertEquals(0, filterResult.messagesRead());
+            assertEquals(records.sizeInBytes(), filterResult.bytesRead());
+            assertEquals(baseOffset, filterResult.maxOffset());
+            assertEquals(0, filterResult.messagesRetained());
+            assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filterResult.bytesRetained());
+            assertEquals(timestamp, filterResult.maxTimestamp());
+            assertEquals(baseOffset, filterResult.shallowOffsetOfMaxTimestamp());
+            assertTrue(filterResult.outputBuffer().position() > 0);
+
+            // Verify filtered records
+            filtered.flip();
+            MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
+            assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filteredRecords.sizeInBytes());
+        }
+    }
+
     @Test
     public void testEmptyBatchDeletion() {
         if (magic >= RecordBatch.MAGIC_VALUE_V2) {
@@ -304,25 +366,32 @@ public void testEmptyBatchDeletion() {
                 long baseOffset = 3L;
                 int baseSequence = 10;
                 int partitionLeaderEpoch = 293;
+                long timestamp = System.currentTimeMillis();
 
                 DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, producerId, producerEpoch,
                         baseSequence, baseOffset, baseOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME,
-                        System.currentTimeMillis(), false, false);
+                        timestamp, false, false);
                 buffer.flip();
 
                 ByteBuffer filtered = ByteBuffer.allocate(2048);
-                MemoryRecords.readableRecords(buffer).filterTo(new TopicPartition("foo", 0), new MemoryRecords.RecordFilter() {
-                    @Override
-                    protected BatchRetention checkBatchRetention(RecordBatch batch) {
-                        return deleteRetention;
-                    }
-
-                    @Override
-                    protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
-                        return false;
-                    }
-                }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
-
+                MemoryRecords records = MemoryRecords.readableRecords(buffer);
+                MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0),
+                        new MemoryRecords.RecordFilter() {
+                            @Override
+                            protected BatchRetention checkBatchRetention(RecordBatch batch) {
+                                return deleteRetention;
+                            }
+
+                            @Override
+                            protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
+                                return false;
+                            }
+                        }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
+
+                // Verify filter result
+                assertEquals(0, filterResult.outputBuffer().position());
+
+                // Verify filtered records
                 filtered.flip();
                 MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
                 assertEquals(0, filteredRecords.sizeInBytes());
@@ -591,15 +660,15 @@ public void testFilterToWithUndersizedBuffer() {
 
             MemoryRecords.FilterResult result = MemoryRecords.readableRecords(buffer)
                     .filterTo(new TopicPartition("foo", 0), new RetainNonNullKeysFilter(), output, Integer.MAX_VALUE,
-                              BufferSupplier.NO_CACHING);
+                            BufferSupplier.NO_CACHING);
 
-            buffer.position(buffer.position() + result.bytesRead);
-            result.output.flip();
+            buffer.position(buffer.position() + result.bytesRead());
+            result.outputBuffer().flip();
 
-            if (output != result.output)
+            if (output != result.outputBuffer())
                 assertEquals(0, output.position());
 
-            MemoryRecords filtered = MemoryRecords.readableRecords(result.output);
+            MemoryRecords filtered = MemoryRecords.readableRecords(result.outputBuffer());
             records.addAll(TestUtils.toList(filtered.records()));
         }
 
@@ -623,9 +692,9 @@ public void testToString() {
                 break;
             case RecordBatch.MAGIC_VALUE_V1:
                 assertEquals("[(record=LegacyRecordBatch(offset=0, Record(magic=1, attributes=0, compression=NONE, " +
-                        "crc=97210616, CreateTime=1000000, key=4 bytes, value=6 bytes))), (record=LegacyRecordBatch(offset=1, " +
-                        "Record(magic=1, attributes=0, compression=NONE, crc=3535988507, CreateTime=1000001, key=4 bytes, " +
-                        "value=6 bytes)))]",
+                                "crc=97210616, CreateTime=1000000, key=4 bytes, value=6 bytes))), (record=LegacyRecordBatch(offset=1, " +
+                                "Record(magic=1, attributes=0, compression=NONE, crc=3535988507, CreateTime=1000001, key=4 bytes, " +
+                                "value=6 bytes)))]",
                         memoryRecords.toString());
                 break;
             case RecordBatch.MAGIC_VALUE_V2:
@@ -669,16 +738,16 @@ public void testFilterTo() {
 
         filtered.flip();
 
-        assertEquals(7, result.messagesRead);
-        assertEquals(4, result.messagesRetained);
-        assertEquals(buffer.limit(), result.bytesRead);
-        assertEquals(filtered.limit(), result.bytesRetained);
+        assertEquals(7, result.messagesRead());
+        assertEquals(4, result.messagesRetained());
+        assertEquals(buffer.limit(), result.bytesRead());
+        assertEquals(filtered.limit(), result.bytesRetained());
         if (magic > RecordBatch.MAGIC_VALUE_V0) {
-            assertEquals(20L, result.maxTimestamp);
+            assertEquals(20L, result.maxTimestamp());
             if (compression == CompressionType.NONE && magic < RecordBatch.MAGIC_VALUE_V2)
-                assertEquals(4L, result.shallowOffsetOfMaxTimestamp);
+                assertEquals(4L, result.shallowOffsetOfMaxTimestamp());
             else
-                assertEquals(5L, result.shallowOffsetOfMaxTimestamp);
+                assertEquals(5L, result.shallowOffsetOfMaxTimestamp());
         }
 
         MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index 91ddbf093058d..04b284ccbd687 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -606,7 +606,7 @@ private[log] class Cleaner(val id: Int,
       position += result.bytesRead
 
       // if any messages are to be retained, write them out
-      val outputBuffer = result.output
+      val outputBuffer = result.outputBuffer
       if (outputBuffer.position() > 0) {
         outputBuffer.flip()
         val retained = MemoryRecords.readableRecords(outputBuffer)
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index 0240707ca3b16..73dfa7e39cd17 100755
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -412,42 +412,57 @@ class LogCleanerTest extends JUnitSuite {
     val log = makeLog(config = LogConfig.fromProps(logConfig.originals, logProps))
 
     val producerEpoch = 0.toShort
-    val producerId = 1L
-    val appendProducer = appendTransactionalAsLeader(log, producerId, producerEpoch)
+    val producer1 = appendTransactionalAsLeader(log, 1L, producerEpoch)
+    val producer2 = appendTransactionalAsLeader(log, 2L, producerEpoch)
 
-    appendProducer(Seq(2, 3)) // batch last offset is 1
-    log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false)
+    // [{Producer1: 2, 3}]
+    producer1(Seq(2, 3)) // offsets 0, 1
     log.roll()
 
-    log.appendAsLeader(record(2, 2), leaderEpoch = 0)
-    log.appendAsLeader(record(3, 3), leaderEpoch = 0)
+    // [{Producer1: 2, 3}], [{Producer2: 2, 3}, {Producer2: Commit}]
+    producer2(Seq(2, 3)) // offsets 2, 3
+    log.appendAsLeader(commitMarker(2L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 4
+    log.roll()
+
+    // [{Producer1: 2, 3}], [{Producer2: 2, 3}, {Producer2: Commit}], [{2}, {3}, {Producer1: Commit}]
+    //  {0, 1},              {2, 3},            {4},                   {5}, {6}, {7} ==> Offsets
+    log.appendAsLeader(record(2, 2), leaderEpoch = 0) // offset 5
+    log.appendAsLeader(record(3, 3), leaderEpoch = 0) // offset 6
+    log.appendAsLeader(commitMarker(1L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 7
     log.roll()
 
     // first time through the records are removed
+    // Expected State: [{Producer1: EmptyBatch}, {Producer2: EmptyBatch}, {Producer2: Commit}, {2}, {3}]
     var dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3), LogTest.keysInLog(log))
-    assertEquals(List(2, 3, 4), offsetsInLog(log)) // commit marker is retained
-    assertEquals(List(1, 2, 3, 4), lastOffsetsPerBatchInLog(log)) // empty batch is retained
+    assertEquals(List(4, 5, 6), offsetsInLog(log))
+    assertEquals(List(1, 3, 4, 5, 6), lastOffsetsPerBatchInLog(log))
 
     // the empty batch remains if cleaned again because it still holds the last sequence
+    // Expected State: [{Producer1: EmptyBatch}, {Producer2: EmptyBatch}, {Producer2: Commit}, {2}, {3}]
     dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3), LogTest.keysInLog(log))
-    assertEquals(List(2, 3, 4), offsetsInLog(log)) // commit marker is still retained
-    assertEquals(List(1, 2, 3, 4), lastOffsetsPerBatchInLog(log)) // empty batch is retained
+    assertEquals(List(4, 5, 6), offsetsInLog(log))
+    assertEquals(List(1, 3, 4, 5, 6), lastOffsetsPerBatchInLog(log))
 
     // append a new record from the producer to allow cleaning of the empty batch
-    appendProducer(Seq(1))
+    // [{Producer1: EmptyBatch}, {Producer2: EmptyBatch}, {Producer2: Commit}, {2}, {3}] [{Producer2: 1}, {Producer2: Commit}]
+    //  {1},                     {3},                     {4},                 {5}, {6},  {8},            {9} ==> Offsets
+    producer2(Seq(1)) // offset 8
+    log.appendAsLeader(commitMarker(2L, producerEpoch), leaderEpoch = 0, isFromClient = false) // offset 9
     log.roll()
 
+    // Expected State: [{Producer1: EmptyBatch}, {Producer2: Commit}, {2}, {3}, {Producer2: 1}, {Producer2: Commit}]
     dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3, 1), LogTest.keysInLog(log))
-    assertEquals(List(2, 3, 4, 5), offsetsInLog(log)) // commit marker is still retained
-    assertEquals(List(2, 3, 4, 5), lastOffsetsPerBatchInLog(log)) // empty batch should be gone
+    assertEquals(List(4, 5, 6, 8, 9), offsetsInLog(log))
+    assertEquals(List(1, 4, 5, 6, 8, 9), lastOffsetsPerBatchInLog(log))
 
+    // Expected State: [{Producer1: EmptyBatch}, {2}, {3}, {Producer2: 1}, {Producer2: Commit}]
     dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1
     assertEquals(List(2, 3, 1), LogTest.keysInLog(log))
-    assertEquals(List(3, 4, 5), offsetsInLog(log)) // commit marker is gone
-    assertEquals(List(3, 4, 5), lastOffsetsPerBatchInLog(log)) // empty batch is gone
+    assertEquals(List(5, 6, 8, 9), offsetsInLog(log))
+    assertEquals(List(1, 5, 6, 8, 9), lastOffsetsPerBatchInLog(log))
   }
 
   @Test

From 4a5cba87bcd4621b999f7290d17d88d6859afb84 Mon Sep 17 00:00:00 2001
From: Flavien Raynaud 
Date: Sun, 9 Sep 2018 08:05:49 +0100
Subject: [PATCH 0787/1847] KAFKA-7286; Avoid getting stuck loading large
 metadata records (#5500)

If a group metadata record size is higher than offsets.load.buffer.size,
loading offsets and group metadata from __consumer_offsets would hang
forever. This was due to the buffer being too small to fit any message
bigger than the maximum configuration. This patch grows the buffer
as needed so the large records will fit and the loading can move on.
A similar change was made to the logic for state loading in the transaction
coordinator.

Reviewers: John Roesler , lambdaliu , Dhruvil Shah , Jason Gustafson 
---
 .../group/GroupMetadataManager.scala          | 19 ++++++++-
 .../transaction/TransactionStateManager.scala | 16 +++++++-
 .../main/scala/kafka/server/KafkaConfig.scala |  4 +-
 .../group/GroupMetadataManagerTest.scala      | 39 ++++++++++++++++++-
 ...ransactionCoordinatorConcurrencyTest.scala |  2 +
 .../TransactionStateManagerTest.scala         |  2 +
 6 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
index 6bd0a5a0d52d6..940ec716e36e8 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala
@@ -510,7 +510,9 @@ class GroupMetadataManager(brokerId: Int,
 
       case Some(log) =>
         var currOffset = log.logStartOffset
-        lazy val buffer = ByteBuffer.allocate(config.loadBufferSize)
+
+        // buffer may not be needed if records are read from memory
+        var buffer = ByteBuffer.allocate(0)
 
         // loop breaks if leader changes at any time during the load, since getHighWatermark is -1
         val loadedOffsets = mutable.Map[GroupTopicPartition, CommitRecordMetadataAndOffset]()
@@ -524,7 +526,20 @@ class GroupMetadataManager(brokerId: Int,
           val memRecords = fetchDataInfo.records match {
             case records: MemoryRecords => records
             case fileRecords: FileRecords =>
-              buffer.clear()
+              val sizeInBytes = fileRecords.sizeInBytes
+              val bytesNeeded = Math.max(config.loadBufferSize, sizeInBytes)
+
+              // minOneMessage = true in the above log.read means that the buffer may need to be grown to ensure progress can be made
+              if (buffer.capacity < bytesNeeded) {
+                if (config.loadBufferSize < bytesNeeded)
+                  warn(s"Loaded offsets and group metadata from $topicPartition with buffer larger ($bytesNeeded bytes) than " +
+                    s"configured offsets.load.buffer.size (${config.loadBufferSize} bytes)")
+
+                buffer = ByteBuffer.allocate(bytesNeeded)
+              } else {
+                buffer.clear()
+              }
+
               fileRecords.readInto(buffer, 0)
               MemoryRecords.readableRecords(buffer)
           }
diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
index a358515445197..50d96c307343e 100644
--- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
+++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala
@@ -296,7 +296,8 @@ class TransactionStateManager(brokerId: Int,
         warn(s"Attempted to load offsets and group metadata from $topicPartition, but found no log")
 
       case Some(log) =>
-        lazy val buffer = ByteBuffer.allocate(config.transactionLogLoadBufferSize)
+        // buffer may not be needed if records are read from memory
+        var buffer = ByteBuffer.allocate(0)
 
         // loop breaks if leader changes at any time during the load, since getHighWatermark is -1
         var currOffset = log.logStartOffset
@@ -311,6 +312,19 @@ class TransactionStateManager(brokerId: Int,
             val memRecords = fetchDataInfo.records match {
               case records: MemoryRecords => records
               case fileRecords: FileRecords =>
+                val sizeInBytes = fileRecords.sizeInBytes
+                val bytesNeeded = Math.max(config.transactionLogLoadBufferSize, sizeInBytes)
+
+                // minOneMessage = true in the above log.read means that the buffer may need to be grown to ensure progress can be made
+                if (buffer.capacity < bytesNeeded) {
+                  if (config.transactionLogLoadBufferSize < bytesNeeded)
+                    warn(s"Loaded offsets and group metadata from $topicPartition with buffer larger ($bytesNeeded bytes) than " +
+                      s"configured transaction.state.log.load.buffer.size (${config.transactionLogLoadBufferSize} bytes)")
+
+                  buffer = ByteBuffer.allocate(bytesNeeded)
+                } else {
+                  buffer.clear()
+                }
                 buffer.clear()
                 fileRecords.readInto(buffer, 0)
                 MemoryRecords.readableRecords(buffer)
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 753a5b91aa0c6..1bc9707fa0bb0 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -660,7 +660,7 @@ object KafkaConfig {
   val GroupInitialRebalanceDelayMsDoc = "The amount of time the group coordinator will wait for more consumers to join a new group before performing the first rebalance. A longer delay means potentially fewer rebalances, but increases the time until processing begins."
   /** ********* Offset management configuration ***********/
   val OffsetMetadataMaxSizeDoc = "The maximum size for a metadata entry associated with an offset commit"
-  val OffsetsLoadBufferSizeDoc = "Batch size for reading from the offsets segments when loading offsets into the cache."
+  val OffsetsLoadBufferSizeDoc = "Batch size for reading from the offsets segments when loading offsets into the cache (soft-limit, overridden if records are too large)."
   val OffsetsTopicReplicationFactorDoc = "The replication factor for the offsets topic (set higher to ensure availability). " +
   "Internal topic creation will fail until the cluster size meets this replication factor requirement."
   val OffsetsTopicPartitionsDoc = "The number of partitions for the offset commit topic (should not change after deployment)"
@@ -677,7 +677,7 @@ object KafkaConfig {
   val TransactionsMaxTimeoutMsDoc = "The maximum allowed timeout for transactions. " +
     "If a client’s requested transaction time exceed this, then the broker will return an error in InitProducerIdRequest. This prevents a client from too large of a timeout, which can stall consumers reading from topics included in the transaction."
   val TransactionsTopicMinISRDoc = "Overridden " + MinInSyncReplicasProp + " config for the transaction topic."
-  val TransactionsLoadBufferSizeDoc = "Batch size for reading from the transaction log segments when loading producer ids and transactions into the cache."
+  val TransactionsLoadBufferSizeDoc = "Batch size for reading from the transaction log segments when loading producer ids and transactions into the cache (soft-limit, overridden if records are too large)."
   val TransactionsTopicReplicationFactorDoc = "The replication factor for the transaction topic (set higher to ensure availability). " +
     "Internal topic creation will fail until the cluster size meets this replication factor requirement."
   val TransactionsTopicPartitionsDoc = "The number of partitions for the transaction topic (should not change after deployment)."
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
index 3e462dba963a1..a0dfbda082e21 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala
@@ -645,6 +645,38 @@ class GroupMetadataManagerTest {
     assertEquals(None, groupMetadataManager.getGroup(groupId))
   }
 
+  @Test
+  def testLoadGroupWithLargeGroupMetadataRecord() {
+    val groupMetadataTopicPartition = groupTopicPartition
+    val startOffset = 15L
+    val committedOffsets = Map(
+      new TopicPartition("foo", 0) -> 23L,
+      new TopicPartition("foo", 1) -> 455L,
+      new TopicPartition("bar", 0) -> 8992L
+    )
+
+    // create a GroupMetadata record larger then offsets.load.buffer.size (here at least 16 bytes larger)
+    val assignmentSize = OffsetConfig.DefaultLoadBufferSize + 16
+    val memberId = "98098230493"
+
+    val offsetCommitRecords = createCommittedOffsetRecords(committedOffsets)
+    val groupMetadataRecord = buildStableGroupRecordWithMember(generation = 15,
+      protocolType = "consumer", protocol = "range", memberId, assignmentSize)
+    val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE,
+      offsetCommitRecords ++ Seq(groupMetadataRecord): _*)
+
+    expectGroupMetadataLoad(groupMetadataTopicPartition, startOffset, records)
+
+    EasyMock.replay(replicaManager)
+
+    groupMetadataManager.loadGroupsAndOffsets(groupMetadataTopicPartition, _ => ())
+
+    val group = groupMetadataManager.getGroup(groupId).getOrElse(fail("Group was not loaded into the cache"))
+    committedOffsets.foreach { case (topicPartition, offset) =>
+      assertEquals(Some(offset), group.offset(topicPartition).map(_.offset))
+    }
+  }
+
   @Test
   def testOffsetWriteAfterGroupRemoved(): Unit = {
     // this test case checks the following scenario:
@@ -1603,7 +1635,7 @@ class GroupMetadataManagerTest {
     val apiVersion = KAFKA_1_1_IV0
     val offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, apiVersion = apiVersion, retentionTime = Some(100))
     val memberId = "98098230493"
-    val groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId, apiVersion)
+    val groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId, apiVersion = apiVersion)
     val records = MemoryRecords.withRecords(startOffset, CompressionType.NONE,
       offsetCommitRecords ++ Seq(groupMetadataRecord): _*)
 
@@ -1708,13 +1740,14 @@ class GroupMetadataManagerTest {
                                                protocolType: String,
                                                protocol: String,
                                                memberId: String,
+                                               assignmentSize: Int = 0,
                                                apiVersion: ApiVersion = ApiVersion.latestVersion): SimpleRecord = {
     val memberProtocols = List((protocol, Array.emptyByteArray))
     val member = new MemberMetadata(memberId, groupId, "clientId", "clientHost", 30000, 10000, protocolType, memberProtocols)
     val group = GroupMetadata.loadGroup(groupId, Stable, generation, protocolType, protocol, memberId,
       if (apiVersion >= KAFKA_2_1_IV0) Some(time.milliseconds()) else None, Seq(member), time)
     val groupMetadataKey = GroupMetadataManager.groupMetadataKey(groupId)
-    val groupMetadataValue = GroupMetadataManager.groupMetadataValue(group, Map(memberId -> Array.empty[Byte]), apiVersion)
+    val groupMetadataValue = GroupMetadataManager.groupMetadataValue(group, Map(memberId -> new Array[Byte](assignmentSize)), apiVersion)
     new SimpleRecord(groupMetadataKey, groupMetadataValue)
   }
 
@@ -1751,6 +1784,8 @@ class GroupMetadataManagerTest {
       EasyMock.eq(true), EasyMock.eq(IsolationLevel.READ_UNCOMMITTED)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
 
+    EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
+
     val bufferCapture = EasyMock.newCapture[ByteBuffer]
     fileRecordsMock.readInto(EasyMock.capture(bufferCapture), EasyMock.anyInt())
     EasyMock.expectLastCall().andAnswer(new IAnswer[Unit] {
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
index 873b88d110473..060e07e732788 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala
@@ -259,6 +259,8 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren
       EasyMock.eq(true), EasyMock.eq(IsolationLevel.READ_UNCOMMITTED)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
 
+    EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
+
     val bufferCapture = EasyMock.newCapture[ByteBuffer]
     fileRecordsMock.readInto(EasyMock.capture(bufferCapture), EasyMock.anyInt())
     EasyMock.expectLastCall().andAnswer(new IAnswer[Unit] {
diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
index 34b82d9ea559f..74bbe336b3ce3 100644
--- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala
@@ -586,6 +586,8 @@ class TransactionStateManagerTest {
       EasyMock.eq(true), EasyMock.eq(IsolationLevel.READ_UNCOMMITTED)))
       .andReturn(FetchDataInfo(LogOffsetMetadata(startOffset), fileRecordsMock))
 
+    EasyMock.expect(fileRecordsMock.sizeInBytes()).andStubReturn(records.sizeInBytes)
+
     val bufferCapture = EasyMock.newCapture[ByteBuffer]
     fileRecordsMock.readInto(EasyMock.capture(bufferCapture), EasyMock.anyInt())
     EasyMock.expectLastCall().andAnswer(new IAnswer[Unit] {

From 05ba5aa00847b18b74369a821e972bbba9f155eb Mon Sep 17 00:00:00 2001
From: Jason Gustafson 
Date: Sun, 9 Sep 2018 00:14:57 -0700
Subject: [PATCH 0788/1847] KAFKA-7333; Protocol changes for KIP-320

This patch contains the protocol updates needed for KIP-320 as well as some of the basic consumer APIs (e.g. `OffsetAndMetadata` and `ConsumerRecord`). The inter-broker format version has not been changed and the brokers will continue to use the current API versions.

Author: Jason Gustafson 

Reviewers: Dong Lin 

Closes #5564 from hachikuji/KAFKA-7333
---
 .../kafka/clients/admin/KafkaAdminClient.java |  11 +-
 .../clients/consumer/ConsumerRecord.java      |  54 ++-
 .../clients/consumer/OffsetAndMetadata.java   |  51 ++-
 .../clients/consumer/OffsetAndTimestamp.java  |  47 ++-
 .../clients/consumer/StickyAssignor.java      |   2 +-
 .../internals/ConsumerCoordinator.java        |   6 +-
 .../consumer/internals/ConsumerProtocol.java  |   2 +-
 .../clients/consumer/internals/Fetcher.java   |  60 +--
 .../internals/TransactionManager.java         |   3 +-
 .../apache/kafka/common/PartitionInfo.java    |   8 +-
 .../apache/kafka/common/TopicPartition.java   |   1 +
 .../kafka/common/protocol/CommonFields.java   |  16 +-
 .../kafka/common/protocol/types/Field.java    |  37 ++
 .../kafka/common/protocol/types/Struct.java   |  54 +++
 .../common/requests/AbstractRequest.java      |   5 +-
 .../requests/AddOffsetsToTxnRequest.java      |   4 +-
 .../requests/AddPartitionsToTxnRequest.java   |   6 +-
 .../requests/AddPartitionsToTxnResponse.java  |   2 +-
 .../common/requests/AlterConfigsRequest.java  |   4 +-
 .../requests/AlterReplicaLogDirsRequest.java  |   6 +-
 .../requests/AlterReplicaLogDirsResponse.java |   2 +-
 .../common/requests/ApiVersionsRequest.java   |   2 +-
 .../requests/ControlledShutdownRequest.java   |   4 +-
 .../common/requests/CreateAclsRequest.java    |   4 +-
 .../CreateDelegationTokenRequest.java         |   4 +-
 .../requests/CreatePartitionsRequest.java     |   4 +-
 .../common/requests/CreateTopicsRequest.java  |   4 +-
 .../common/requests/DeleteAclsRequest.java    |   4 +-
 .../common/requests/DeleteGroupsRequest.java  |   4 +-
 .../common/requests/DeleteRecordsRequest.java |   6 +-
 .../requests/DeleteRecordsResponse.java       |   2 +-
 .../common/requests/DeleteTopicsRequest.java  |   4 +-
 .../common/requests/DescribeAclsRequest.java  |   4 +-
 .../requests/DescribeConfigsRequest.java      |   4 +-
 .../DescribeDelegationTokenRequest.java       |   4 +-
 .../requests/DescribeGroupsRequest.java       |   4 +-
 .../requests/DescribeLogDirsRequest.java      |   4 +-
 .../requests/DescribeLogDirsResponse.java     |   2 +-
 .../kafka/common/requests/EndTxnRequest.java  |   4 +-
 .../kafka/common/requests/EpochEndOffset.java |   1 -
 .../ExpireDelegationTokenRequest.java         |   4 +-
 .../kafka/common/requests/FetchRequest.java   | 352 +++++++++---------
 .../kafka/common/requests/FetchResponse.java  | 127 +++----
 .../requests/FindCoordinatorRequest.java      |   4 +-
 .../common/requests/HeartbeatRequest.java     |   4 +-
 .../requests/InitProducerIdRequest.java       |   4 +-
 .../common/requests/JoinGroupRequest.java     |   4 +-
 .../common/requests/LeaderAndIsrRequest.java  |   4 +-
 .../common/requests/LeaveGroupRequest.java    |   4 +-
 .../common/requests/ListGroupsRequest.java    |   4 +-
 .../common/requests/ListOffsetRequest.java    | 282 +++++++-------
 .../common/requests/ListOffsetResponse.java   | 156 +++++---
 .../common/requests/MetadataRequest.java      |  36 +-
 .../common/requests/MetadataResponse.java     | 319 ++++++++--------
 .../common/requests/OffsetCommitRequest.java  | 205 +++++-----
 .../common/requests/OffsetCommitResponse.java |  83 +++--
 .../common/requests/OffsetFetchRequest.java   |  75 ++--
 .../common/requests/OffsetFetchResponse.java  | 129 ++++---
 .../OffsetsForLeaderEpochRequest.java         | 102 +++--
 .../OffsetsForLeaderEpochResponse.java        |  61 +--
 .../kafka/common/requests/ProduceRequest.java |   6 +-
 .../common/requests/ProduceResponse.java      |   2 +-
 .../requests/RenewDelegationTokenRequest.java |   4 +-
 .../kafka/common/requests/RequestUtils.java   |  16 +
 .../requests/SaslAuthenticateRequest.java     |   4 +-
 .../common/requests/SaslHandshakeRequest.java |   4 +-
 .../common/requests/StopReplicaRequest.java   |   4 +-
 .../common/requests/SyncGroupRequest.java     |   4 +-
 .../requests/TxnOffsetCommitRequest.java      | 107 +++---
 .../requests/TxnOffsetCommitResponse.java     |  69 ++--
 .../requests/UpdateMetadataRequest.java       |   4 +-
 .../requests/WriteTxnMarkersRequest.java      |   6 +-
 .../requests/WriteTxnMarkersResponse.java     |   2 +-
 .../kafka/common/utils/CollectionUtils.java   |  29 +-
 .../clients/FetchSessionHandlerTest.java      |  73 ++--
 .../clients/admin/KafkaAdminClientTest.java   |  40 +-
 .../clients/consumer/ConsumerRecordTest.java  |   3 +
 .../clients/consumer/KafkaConsumerTest.java   |  22 +-
 ...taTest.java => OffsetAndMetadataTest.java} |  46 +--
 .../clients/consumer/StickyAssignorTest.java  |   4 +-
 .../internals/ConsumerCoordinatorTest.java    |  44 ++-
 .../consumer/internals/FetcherTest.java       |  99 ++++-
 .../internals/TransactionManagerTest.java     |  22 +-
 ...itionTest.java => TopicPartitionTest.java} |   3 +-
 .../common/requests/RequestResponseTest.java  | 118 +++---
 ...ile => offsetAndMetadataBeforeLeaderEpoch} | Bin
 .../offsetAndMetadataWithLeaderEpoch          | Bin 0 -> 257 bytes
 .../src/main/scala/kafka/api/ApiVersion.scala |  11 +-
 .../group/GroupMetadataManager.scala          |  18 +-
 .../scala/kafka/server/FetchSession.scala     |   3 +-
 .../main/scala/kafka/server/KafkaApis.scala   |  45 ++-
 .../scala/kafka/server/MetadataCache.scala    |  14 +-
 .../server/ReplicaAlterLogDirsThread.scala    |   4 +-
 .../kafka/server/ReplicaFetcherThread.scala   |  38 +-
 .../scala/kafka/server/ReplicaManager.scala   |   6 +-
 .../kafka/tools/ReplicaVerificationTool.scala |   5 +-
 .../kafka/api/AuthorizerIntegrationTest.scala |  13 +-
 .../scala/unit/kafka/api/ApiVersionTest.scala |   3 +-
 .../server/AbstractFetcherThreadTest.scala    |   4 +-
 ...FetchRequestDownConversionConfigTest.scala |   5 +-
 .../unit/kafka/server/FetchRequestTest.scala  |   5 +-
 .../unit/kafka/server/FetchSessionTest.scala  |  29 +-
 .../unit/kafka/server/KafkaApisTest.scala     |  18 +-
 .../kafka/server/ListOffsetsRequestTest.scala |  10 +-
 .../unit/kafka/server/LogOffsetTest.scala     |  16 +-
 .../unit/kafka/server/MetadataCacheTest.scala |   3 +-
 .../OffsetsForLeaderEpochRequestTest.scala    |   4 +-
 .../server/ReplicaFetcherThreadTest.scala     |   2 +-
 .../server/ReplicaManagerQuotasTest.scala     |  12 +-
 .../kafka/server/ReplicaManagerTest.scala     |  57 ++-
 .../unit/kafka/server/RequestQuotaTest.scala  |  15 +-
 .../unit/kafka/server/SimpleFetchTest.scala   |   5 +-
 .../epoch/LeaderEpochIntegrationTest.scala    |  11 +-
 .../epoch/OffsetsForLeaderEpochTest.scala     |  10 +-
 114 files changed, 2062 insertions(+), 1448 deletions(-)
 rename clients/src/test/java/org/apache/kafka/clients/consumer/{SerializeCompatibilityOffsetAndMetadataTest.java => OffsetAndMetadataTest.java} (50%)
 rename clients/src/test/java/org/apache/kafka/common/{SerializeCompatibilityTopicPartitionTest.java => TopicPartitionTest.java} (98%)
 rename clients/src/test/resources/serializedData/{offsetAndMetadataSerializedfile => offsetAndMetadataBeforeLeaderEpoch} (100%)
 create mode 100644 clients/src/test/resources/serializedData/offsetAndMetadataWithLeaderEpoch

diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index 904cd0601e5b1..5759d632837c3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -134,6 +134,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -2643,12 +2644,14 @@ void handleResponse(AbstractResponse abstractResponse) {
                             for (Map.Entry entry :
                                     response.responseData().entrySet()) {
                                 final TopicPartition topicPartition = entry.getKey();
-                                final Errors error = entry.getValue().error;
+                                OffsetFetchResponse.PartitionData partitionData = entry.getValue();
+                                final Errors error = partitionData.error;
 
                                 if (error == Errors.NONE) {
-                                    final Long offset = entry.getValue().offset;
-                                    final String metadata = entry.getValue().metadata;
-                                    groupOffsetsListing.put(topicPartition, new OffsetAndMetadata(offset, metadata));
+                                    final Long offset = partitionData.offset;
+                                    final String metadata = partitionData.metadata;
+                                    final Optional leaderEpoch = partitionData.leaderEpoch;
+                                    groupOffsetsListing.put(topicPartition, new OffsetAndMetadata(offset, leaderEpoch, metadata));
                                 } else {
                                     log.warn("Skipping return offset for {} due to error {}.", topicPartition, error);
                                 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecord.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecord.java
index 7f85246184427..0413d5b07768e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecord.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecord.java
@@ -22,6 +22,8 @@
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.record.TimestampType;
 
+import java.util.Optional;
+
 /**
  * A key/value pair to be received from Kafka. This also consists of a topic name and 
  * a partition number from which the record is being received, an offset that points 
@@ -42,6 +44,7 @@ public class ConsumerRecord {
     private final Headers headers;
     private final K key;
     private final V value;
+    private final Optional leaderEpoch;
 
     private volatile Long checksum;
 
@@ -120,8 +123,41 @@ public ConsumerRecord(String topic,
                           K key,
                           V value,
                           Headers headers) {
+        this(topic, partition, offset, timestamp, timestampType, checksum, serializedKeySize, serializedValueSize,
+                key, value, headers, Optional.empty());
+    }
+
+    /**
+     * Creates a record to be received from a specified topic and partition
+     *
+     * @param topic The topic this record is received from
+     * @param partition The partition of the topic this record is received from
+     * @param offset The offset of this record in the corresponding Kafka partition
+     * @param timestamp The timestamp of the record.
+     * @param timestampType The timestamp type
+     * @param checksum The checksum (CRC32) of the full record
+     * @param serializedKeySize The length of the serialized key
+     * @param serializedValueSize The length of the serialized value
+     * @param key The key of the record, if one exists (null is allowed)
+     * @param value The record contents
+     * @param headers The headers of the record
+     * @param leaderEpoch Optional leader epoch of the record (may be empty for legacy record formats)
+     */
+    public ConsumerRecord(String topic,
+                          int partition,
+                          long offset,
+                          long timestamp,
+                          TimestampType timestampType,
+                          Long checksum,
+                          int serializedKeySize,
+                          int serializedValueSize,
+                          K key,
+                          V value,
+                          Headers headers,
+                          Optional leaderEpoch) {
         if (topic == null)
             throw new IllegalArgumentException("Topic cannot be null");
+
         this.topic = topic;
         this.partition = partition;
         this.offset = offset;
@@ -133,6 +169,7 @@ public ConsumerRecord(String topic,
         this.key = key;
         this.value = value;
         this.headers = headers;
+        this.leaderEpoch = leaderEpoch;
     }
 
     /**
@@ -225,13 +262,26 @@ public int serializedValueSize() {
         return this.serializedValueSize;
     }
 
+    /**
+     * Get the leader epoch for the record if available
+     *
+     * @return the leader epoch or empty for legacy record formats
+     */
+    public Optional leaderEpoch() {
+        return leaderEpoch;
+    }
+
     @Override
     public String toString() {
-        return "ConsumerRecord(topic = " + topic() + ", partition = " + partition() + ", offset = " + offset()
+        return "ConsumerRecord(topic = " + topic
+               + ", partition = " + partition
+               + ", leaderEpoch = " + leaderEpoch.orElse(null)
+               + ", offset = " + offset
                + ", " + timestampType + " = " + timestamp
                + ", serialized key size = "  + serializedKeySize
                + ", serialized value size = " + serializedValueSize
                + ", headers = " + headers
-               + ", key = " + key + ", value = " + value + ")";
+               + ", key = " + key
+               + ", value = " + value + ")";
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java
index 262d8f8fc6c25..aa91e509b8759 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java
@@ -19,6 +19,8 @@
 import org.apache.kafka.common.requests.OffsetFetchResponse;
 
 import java.io.Serializable;
+import java.util.Objects;
+import java.util.Optional;
 
 /**
  * The Kafka offset commit API allows users to provide additional metadata (in the form of a string)
@@ -26,16 +28,30 @@
  * node made the commit, what time the commit was made, etc.
  */
 public class OffsetAndMetadata implements Serializable {
+    private static final long serialVersionUID = 2019555404968089681L;
+
     private final long offset;
     private final String metadata;
 
+    // We use null to represent the absence of a leader epoch to simplify serialization.
+    // I.e., older serializations of this class which do not have this field will automatically
+    // initialize its value to null.
+    private final Integer leaderEpoch;
+
     /**
      * Construct a new OffsetAndMetadata object for committing through {@link KafkaConsumer}.
+     *
      * @param offset The offset to be committed
+     * @param leaderEpoch Optional leader epoch of the last consumed record
      * @param metadata Non-null metadata
      */
-    public OffsetAndMetadata(long offset, String metadata) {
+    public OffsetAndMetadata(long offset, Optional leaderEpoch, String metadata) {
+        if (offset < 0)
+            throw new IllegalArgumentException("Invalid negative offset");
+
         this.offset = offset;
+        this.leaderEpoch = leaderEpoch.orElse(null);
+
         // The server converts null metadata to an empty string. So we store it as an empty string as well on the client
         // to be consistent.
         if (metadata == null)
@@ -44,6 +60,15 @@ public OffsetAndMetadata(long offset, String metadata) {
             this.metadata = metadata;
     }
 
+    /**
+     * Construct a new OffsetAndMetadata object for committing through {@link KafkaConsumer}.
+     * @param offset The offset to be committed
+     * @param metadata Non-null metadata
+     */
+    public OffsetAndMetadata(long offset, String metadata) {
+        this(offset, Optional.empty(), metadata);
+    }
+
     /**
      * Construct a new OffsetAndMetadata object for committing through {@link KafkaConsumer}. The metadata
      * associated with the commit will be empty.
@@ -61,29 +86,39 @@ public String metadata() {
         return metadata;
     }
 
+    /**
+     * Get the leader epoch of the previously consumed record (if one is known). Log truncation is detected
+     * if there exists a leader epoch which is larger than this epoch and begins at an offset earlier than
+     * the committed offset.
+     *
+     * @return the leader epoch or empty if not known
+     */
+    public Optional leaderEpoch() {
+        return Optional.ofNullable(leaderEpoch);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         OffsetAndMetadata that = (OffsetAndMetadata) o;
-
-        if (offset != that.offset) return false;
-        return metadata.equals(that.metadata);
+        return offset == that.offset &&
+                Objects.equals(metadata, that.metadata) &&
+                Objects.equals(leaderEpoch, that.leaderEpoch);
     }
 
     @Override
     public int hashCode() {
-        int result = (int) (offset ^ (offset >>> 32));
-        result = 31 * result + metadata.hashCode();
-        return result;
+        return Objects.hash(offset, metadata, leaderEpoch);
     }
 
     @Override
     public String toString() {
         return "OffsetAndMetadata{" +
                 "offset=" + offset +
+                ", leaderEpoch=" + leaderEpoch +
                 ", metadata='" + metadata + '\'' +
                 '}';
     }
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndTimestamp.java b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndTimestamp.java
index 3af057f9ce4ae..40d993074a25e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndTimestamp.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndTimestamp.java
@@ -16,7 +16,8 @@
  */
 package org.apache.kafka.clients.consumer;
 
-import org.apache.kafka.common.utils.Utils;
+import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A container class for offset and timestamp.
@@ -24,12 +25,22 @@
 public final class OffsetAndTimestamp {
     private final long timestamp;
     private final long offset;
+    private final Optional leaderEpoch;
 
     public OffsetAndTimestamp(long offset, long timestamp) {
+        this(offset, timestamp, Optional.empty());
+    }
+
+    public OffsetAndTimestamp(long offset, long timestamp, Optional leaderEpoch) {
+        if (offset < 0)
+            throw new IllegalArgumentException("Invalid negative offset");
+
+        if (timestamp < 0)
+            throw new IllegalArgumentException("Invalid negative timestamp");
+
         this.offset = offset;
-        assert this.offset >= 0;
         this.timestamp = timestamp;
-        assert this.timestamp >= 0;
+        this.leaderEpoch = leaderEpoch;
     }
 
     public long timestamp() {
@@ -40,21 +51,35 @@ public long offset() {
         return offset;
     }
 
+    /**
+     * Get the leader epoch corresponding to the offset that was found (if one exists).
+     * This can be provided to seek() to ensure that the log hasn't been truncated prior to fetching.
+     *
+     * @return The leader epoch or empty if it is not known
+     */
+    public Optional leaderEpoch() {
+        return leaderEpoch;
+    }
+
     @Override
     public String toString() {
-        return "(timestamp=" + timestamp + ", offset=" + offset + ")";
+        return "(timestamp=" + timestamp +
+                ", leaderEpoch=" + leaderEpoch.orElse(null) +
+                ", offset=" + offset + ")";
     }
 
     @Override
-    public int hashCode() {
-        return 31 * Utils.longHashcode(timestamp) + Utils.longHashcode(offset);
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        OffsetAndTimestamp that = (OffsetAndTimestamp) o;
+        return timestamp == that.timestamp &&
+                offset == that.offset &&
+                Objects.equals(leaderEpoch, that.leaderEpoch);
     }
 
     @Override
-    public boolean equals(Object o) {
-        if (o == null || !(o instanceof OffsetAndTimestamp))
-            return false;
-        OffsetAndTimestamp other = (OffsetAndTimestamp) o;
-        return this.timestamp == other.timestamp() && this.offset == other.offset();
+    public int hashCode() {
+        return Objects.hash(timestamp, offset, leaderEpoch);
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java
index 12da7e5367dd5..0d74eed7835fe 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/StickyAssignor.java
@@ -671,7 +671,7 @@ boolean isSticky() {
     static ByteBuffer serializeTopicPartitionAssignment(List partitions) {
         Struct struct = new Struct(STICKY_ASSIGNOR_USER_DATA);
         List topicAssignments = new ArrayList<>();
-        for (Map.Entry> topicEntry : CollectionUtils.groupDataByTopic(partitions).entrySet()) {
+        for (Map.Entry> topicEntry : CollectionUtils.groupPartitionsByTopic(partitions).entrySet()) {
             Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT);
             topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey());
             topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray());
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index ce2db3566e9bc..3b0a5fa9c0aba 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -760,8 +760,8 @@ private RequestFuture sendOffsetCommitRequest(final Map= 0) {
                     // record the position with the offset (-1 indicates no committed offset to fetch)
-                    offsets.put(tp, new OffsetAndMetadata(data.offset, data.metadata));
+                    offsets.put(tp, new OffsetAndMetadata(data.offset, data.leaderEpoch, data.metadata));
                 } else {
                     log.debug("Found no committed offset for partition {}", tp);
                 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index 920c2957c4dc9..8a4aef8b33487 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -123,7 +123,7 @@ public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assign
         Struct struct = new Struct(ASSIGNMENT_V0);
         struct.set(USER_DATA_KEY_NAME, assignment.userData());
         List topicAssignments = new ArrayList<>();
-        Map> partitionsByTopic = CollectionUtils.groupDataByTopic(assignment.partitions());
+        Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(assignment.partitions());
         for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) {
             Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0);
             topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey());
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index 36a3314e69832..dc0daa233abb6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -82,6 +82,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -168,10 +169,12 @@ public Fetcher(LogContext logContext,
     private static class OffsetData {
         final long offset;
         final Long timestamp; //  null if the broker does not support returning timestamps
+        final Optional leaderEpoch; // empty if the leader epoch is not known
 
-        OffsetData(long offset, Long timestamp) {
+        OffsetData(long offset, Long timestamp, Optional leaderEpoch) {
             this.offset = offset;
             this.timestamp = timestamp;
+            this.leaderEpoch = leaderEpoch;
         }
     }
 
@@ -383,7 +386,8 @@ public Map offsetsByTimes(Map partitionResetTimestamp
         for (TopicPartition tp : partitionResetTimestamps.keySet())
             metadata.add(tp.topic());
 
-        Map> timestampsToSearchByNode = groupListOffsetRequests(partitionResetTimestamps);
-        for (Map.Entry> entry : timestampsToSearchByNode.entrySet()) {
+        Map> timestampsToSearchByNode =
+                groupListOffsetRequests(partitionResetTimestamps);
+        for (Map.Entry> entry : timestampsToSearchByNode.entrySet()) {
             Node node = entry.getKey();
-            final Map resetTimestamps = entry.getValue();
+            final Map resetTimestamps = entry.getValue();
             subscriptions.setResetPending(resetTimestamps.keySet(), time.milliseconds() + requestTimeoutMs);
 
             RequestFuture future = sendListOffsetRequest(node, resetTimestamps, false);
@@ -588,8 +593,8 @@ public void onSuccess(ListOffsetResult result) {
                     for (Map.Entry fetchedOffset : result.fetchedOffsets.entrySet()) {
                         TopicPartition partition = fetchedOffset.getKey();
                         OffsetData offsetData = fetchedOffset.getValue();
-                        Long requestedResetTimestamp = resetTimestamps.get(partition);
-                        resetOffsetIfNeeded(partition, requestedResetTimestamp, offsetData);
+                        ListOffsetRequest.PartitionData requestedReset = resetTimestamps.get(partition);
+                        resetOffsetIfNeeded(partition, requestedReset.timestamp, offsetData);
                     }
                 }
 
@@ -619,7 +624,8 @@ private RequestFuture sendListOffsetsRequests(final Map> timestampsToSearchByNode = groupListOffsetRequests(timestampsToSearch);
+        Map> timestampsToSearchByNode =
+                groupListOffsetRequests(timestampsToSearch);
         if (timestampsToSearchByNode.isEmpty())
             return RequestFuture.failure(new StaleMetadataException());
 
@@ -628,7 +634,7 @@ private RequestFuture sendListOffsetsRequests(final Map partitionsToRetry = new HashSet<>();
         final AtomicInteger remainingResponses = new AtomicInteger(timestampsToSearchByNode.size());
 
-        for (Map.Entry> entry : timestampsToSearchByNode.entrySet()) {
+        for (Map.Entry> entry : timestampsToSearchByNode.entrySet()) {
             RequestFuture future =
                     sendListOffsetRequest(entry.getKey(), entry.getValue(), requireTimestamps);
             future.addListener(new RequestFutureListener() {
@@ -657,8 +663,9 @@ public void onFailure(RuntimeException e) {
         return listOffsetRequestsFuture;
     }
 
-    private Map> groupListOffsetRequests(Map timestampsToSearch) {
-        final Map> timestampsToSearchByNode = new HashMap<>();
+    private Map> groupListOffsetRequests(
+            Map timestampsToSearch) {
+        final Map> timestampsToSearchByNode = new HashMap<>();
         for (Map.Entry entry: timestampsToSearch.entrySet()) {
             TopicPartition tp  = entry.getKey();
             PartitionInfo info = metadata.fetch().partition(tp);
@@ -679,12 +686,11 @@ private Map> groupListOffsetRequests(Map topicData = timestampsToSearchByNode.get(node);
-                if (topicData == null) {
-                    topicData = new HashMap<>();
-                    timestampsToSearchByNode.put(node, topicData);
-                }
-                topicData.put(entry.getKey(), entry.getValue());
+                Map topicData =
+                        timestampsToSearchByNode.computeIfAbsent(node, n -> new HashMap<>());
+                ListOffsetRequest.PartitionData partitionData = new ListOffsetRequest.PartitionData(
+                        entry.getValue(), Optional.empty());
+                topicData.put(entry.getKey(), partitionData);
             }
         }
         return timestampsToSearchByNode;
@@ -699,7 +705,7 @@ private Map> groupListOffsetRequests(Map sendListOffsetRequest(final Node node,
-                                                                  final Map timestampsToSearch,
+                                                                  final Map timestampsToSearch,
                                                                   boolean requireTimestamp) {
         ListOffsetRequest.Builder builder = ListOffsetRequest.Builder
                 .forConsumer(requireTimestamp, isolationLevel)
@@ -729,14 +735,14 @@ public void onSuccess(ClientResponse response, RequestFuture f
      *               return a null timestamp (-1 is returned instead when necessary).
      */
     @SuppressWarnings("deprecation")
-    private void handleListOffsetResponse(Map timestampsToSearch,
+    private void handleListOffsetResponse(Map timestampsToSearch,
                                           ListOffsetResponse listOffsetResponse,
                                           RequestFuture future) {
         Map fetchedOffsets = new HashMap<>();
         Set partitionsToRetry = new HashSet<>();
         Set unauthorizedTopics = new HashSet<>();
 
-        for (Map.Entry entry : timestampsToSearch.entrySet()) {
+        for (Map.Entry entry : timestampsToSearch.entrySet()) {
             TopicPartition topicPartition = entry.getKey();
             ListOffsetResponse.PartitionData partitionData = listOffsetResponse.responseData().get(topicPartition);
             Errors error = partitionData.error;
@@ -756,7 +762,7 @@ private void handleListOffsetResponse(Map timestampsToSear
                     log.debug("Handling v0 ListOffsetResponse response for {}. Fetched offset {}",
                             topicPartition, offset);
                     if (offset != ListOffsetResponse.UNKNOWN_OFFSET) {
-                        OffsetData offsetData = new OffsetData(offset, null);
+                        OffsetData offsetData = new OffsetData(offset, null, Optional.empty());
                         fetchedOffsets.put(topicPartition, offsetData);
                     }
                 } else {
@@ -764,7 +770,8 @@ private void handleListOffsetResponse(Map timestampsToSear
                     log.debug("Handling ListOffsetResponse response for {}. Fetched offset {}, timestamp {}",
                             topicPartition, partitionData.offset, partitionData.timestamp);
                     if (partitionData.offset != ListOffsetResponse.UNKNOWN_OFFSET) {
-                        OffsetData offsetData = new OffsetData(partitionData.offset, partitionData.timestamp);
+                        OffsetData offsetData = new OffsetData(partitionData.offset, partitionData.timestamp,
+                                partitionData.leaderEpoch);
                         fetchedOffsets.put(topicPartition, offsetData);
                     }
                 }
@@ -857,7 +864,7 @@ private Map prepareFetchRequests() {
 
                 long position = this.subscriptions.position(partition);
                 builder.add(partition, new FetchRequest.PartitionData(position, FetchRequest.INVALID_LOG_START_OFFSET,
-                    this.fetchSize));
+                    this.fetchSize, Optional.empty()));
 
                 log.debug("Added {} fetch request for partition {} at offset {} to node {}", isolationLevel,
                     partition, position, node);
@@ -979,6 +986,7 @@ private ConsumerRecord parseRecord(TopicPartition partition,
         try {
             long offset = record.offset();
             long timestamp = record.timestamp();
+            Optional leaderEpoch = maybeLeaderEpoch(batch.partitionLeaderEpoch());
             TimestampType timestampType = batch.timestampType();
             Headers headers = new RecordHeaders(record.headers());
             ByteBuffer keyBytes = record.key();
@@ -991,13 +999,17 @@ private ConsumerRecord parseRecord(TopicPartition partition,
                                         timestamp, timestampType, record.checksumOrNull(),
                                         keyByteArray == null ? ConsumerRecord.NULL_SIZE : keyByteArray.length,
                                         valueByteArray == null ? ConsumerRecord.NULL_SIZE : valueByteArray.length,
-                                        key, value, headers);
+                                        key, value, headers, leaderEpoch);
         } catch (RuntimeException e) {
             throw new SerializationException("Error deserializing key/value for partition " + partition +
                     " at offset " + record.offset() + ". If needed, please seek past the record to continue consumption.", e);
         }
     }
 
+    private Optional maybeLeaderEpoch(int leaderEpoch) {
+        return leaderEpoch == RecordBatch.NO_PARTITION_LEADER_EPOCH ? Optional.empty() : Optional.of(leaderEpoch);
+    }
+
     @Override
     public void onAssignment(Set assignment) {
         sensors.updatePartitionLagAndLeadSensors(assignment);
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
index c0685c9cb2dcf..2cbd1e97d1a6d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@ -839,7 +839,8 @@ private TxnOffsetCommitHandler txnOffsetCommitHandler(TransactionalRequestResult
                                                           String consumerGroupId) {
         for (Map.Entry entry : offsets.entrySet()) {
             OffsetAndMetadata offsetAndMetadata = entry.getValue();
-            CommittedOffset committedOffset = new CommittedOffset(offsetAndMetadata.offset(), offsetAndMetadata.metadata());
+            CommittedOffset committedOffset = new CommittedOffset(offsetAndMetadata.offset(),
+                    offsetAndMetadata.metadata(), offsetAndMetadata.leaderEpoch());
             pendingTxnOffsetCommits.put(entry.getKey(), committedOffset);
         }
         TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(transactionalId, consumerGroupId,
diff --git a/clients/src/main/java/org/apache/kafka/common/PartitionInfo.java b/clients/src/main/java/org/apache/kafka/common/PartitionInfo.java
index 38e4f6788e90f..44cd4f4d215f0 100644
--- a/clients/src/main/java/org/apache/kafka/common/PartitionInfo.java
+++ b/clients/src/main/java/org/apache/kafka/common/PartitionInfo.java
@@ -20,7 +20,6 @@
  * This is used to describe per-partition state in the MetadataResponse.
  */
 public class PartitionInfo {
-
     private final String topic;
     private final int partition;
     private final Node leader;
@@ -33,7 +32,12 @@ public PartitionInfo(String topic, int partition, Node leader, Node[] replicas,
         this(topic, partition, leader, replicas, inSyncReplicas, new Node[0]);
     }
 
-    public PartitionInfo(String topic, int partition, Node leader, Node[] replicas, Node[] inSyncReplicas, Node[] offlineReplicas) {
+    public PartitionInfo(String topic,
+                         int partition,
+                         Node leader,
+                         Node[] replicas,
+                         Node[] inSyncReplicas,
+                         Node[] offlineReplicas) {
         this.topic = topic;
         this.partition = partition;
         this.leader = leader;
diff --git a/clients/src/main/java/org/apache/kafka/common/TopicPartition.java b/clients/src/main/java/org/apache/kafka/common/TopicPartition.java
index dc79c2e13dc90..08b2a51d58943 100644
--- a/clients/src/main/java/org/apache/kafka/common/TopicPartition.java
+++ b/clients/src/main/java/org/apache/kafka/common/TopicPartition.java
@@ -22,6 +22,7 @@
  * A topic name and partition number
  */
 public final class TopicPartition implements Serializable {
+    private static final long serialVersionUID = -613627415771699627L;
 
     private int hash = 0;
     private final int partition;
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java b/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java
index 9eddf2b17e668..708500c65e6ce 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java
@@ -27,7 +27,12 @@ public class CommonFields {
     public static final Field.Int32 PARTITION_ID = new Field.Int32("partition", "Topic partition id");
     public static final Field.Int16 ERROR_CODE = new Field.Int16("error_code", "Response error code");
     public static final Field.NullableStr ERROR_MESSAGE = new Field.NullableStr("error_message", "Response error message");
-    public static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch", "The epoch");
+    public static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch", "The leader epoch");
+    public static final Field.Int32 CURRENT_LEADER_EPOCH = new Field.Int32("current_leader_epoch",
+            "The current leader epoch, if provided, is used to fence consumers/replicas with old metadata. " +
+                    "If the epoch provided by the client is larger than the current epoch known to the broker, then " +
+                    "the UNKNOWN_LEADER_EPOCH error code will be returned. If the provided epoch is smaller, then " +
+                    "the FENCED_LEADER_EPOCH error code will be returned.");
 
     // Group APIs
     public static final Field.Str GROUP_ID = new Field.Str("group_id", "The unique group identifier");
@@ -58,4 +63,13 @@ public class CommonFields {
     public static final Field.Str PRINCIPAL_TYPE = new Field.Str("principal_type", "principalType of the Kafka principal");
     public static final Field.Str PRINCIPAL_NAME = new Field.Str("name", "name of the Kafka principal");
 
+    public static final Field.Int64 COMMITTED_OFFSET = new Field.Int64("offset",
+            "Message offset to be committed");
+    public static final Field.NullableStr COMMITTED_METADATA = new Field.NullableStr("metadata",
+            "Any associated metadata the client wants to keep.");
+    public static final Field.Int32 COMMITTED_LEADER_EPOCH = new Field.Int32("leader_epoch",
+            "The leader epoch, if provided is derived from the last consumed record. " +
+                    "This is used by the consumer to check for log truncation and to ensure partition " +
+                    "metadata is up to date following a group rebalance.");
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
index 5c170017454e2..72e051c179e73 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
@@ -92,4 +92,41 @@ public NullableStr(String name, String docString) {
             super(name, Type.NULLABLE_STRING, docString, false, null);
         }
     }
+
+    public static class Bool extends Field {
+        public Bool(String name, String docString) {
+            super(name, Type.BOOLEAN, docString, false, null);
+        }
+    }
+
+    public static class Array extends Field {
+        public Array(String name, Type elementType, String docString) {
+            super(name, new ArrayOf(elementType), docString, false, null);
+        }
+    }
+
+    public static class ComplexArray {
+        public final String name;
+        public final String docString;
+
+        public ComplexArray(String name, String docString) {
+            this.name = name;
+            this.docString = docString;
+        }
+
+        public Field withFields(Field... fields) {
+            Schema elementType = new Schema(fields);
+            return new Field(name, new ArrayOf(elementType), docString, false, null);
+        }
+
+        public Field nullableWithFields(Field... fields) {
+            Schema elementType = new Schema(fields);
+            return new Field(name, ArrayOf.nullable(elementType), docString, false, null);
+        }
+
+        public Field withFields(String docStringOverride, Field... fields) {
+            Schema elementType = new Schema(fields);
+            return new Field(name, new ArrayOf(elementType), docStringOverride, false, null);
+        }
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index 7183aedbd951a..94d5ae1a79725 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -99,6 +99,14 @@ public String get(Field.NullableStr field) {
         return getString(field.name);
     }
 
+    public Object[] get(Field.Array field) {
+        return getArray(field.name);
+    }
+
+    public Object[] get(Field.ComplexArray field) {
+        return getArray(field.name);
+    }
+
     public Long getOrElse(Field.Int64 field, long alternative) {
         if (hasField(field.name))
             return getLong(field.name);
@@ -135,6 +143,24 @@ public String getOrElse(Field.Str field, String alternative) {
         return alternative;
     }
 
+    public boolean getOrElse(Field.Bool field, boolean alternative) {
+        if (hasField(field.name))
+            return getBoolean(field.name);
+        return alternative;
+    }
+
+    public Object[] getOrEmpty(Field.Array field) {
+        if (hasField(field.name))
+            return getArray(field.name);
+        return new Object[0];
+    }
+
+    public Object[] getOrEmpty(Field.ComplexArray field) {
+        if (hasField(field.name))
+            return getArray(field.name);
+        return new Object[0];
+    }
+
     /**
      * Get the record value for the field with the given name by doing a hash table lookup (slower!)
      *
@@ -162,6 +188,10 @@ public boolean hasField(Field def) {
         return schema.get(def.name) != null;
     }
 
+    public boolean hasField(Field.ComplexArray def) {
+        return schema.get(def.name) != null;
+    }
+
     public Struct getStruct(BoundField field) {
         return (Struct) get(field);
     }
@@ -300,6 +330,22 @@ public Struct set(Field.Int16 def, short value) {
         return set(def.name, value);
     }
 
+    public Struct set(Field.Array def, Object[] value) {
+        return set(def.name, value);
+    }
+
+    public Struct set(Field.ComplexArray def, Object[] value) {
+        return set(def.name, value);
+    }
+
+    public Struct setIfExists(Field.Array def, Object[] value) {
+        return set(def.name, value);
+    }
+
+    public Struct setIfExists(Field.ComplexArray def, Object[] value) {
+        return set(def.name, value);
+    }
+
     public Struct setIfExists(Field def, Object value) {
         return setIfExists(def.name, value);
     }
@@ -343,6 +389,14 @@ public Struct instance(String field) {
         return instance(schema.get(field));
     }
 
+    public Struct instance(Field field) {
+        return instance(schema.get(field.name));
+    }
+
+    public Struct instance(Field.ComplexArray field) {
+        return instance(schema.get(field.name));
+    }
+
     /**
      * Empty all the values from this record
      */
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index d2b93c4590eb9..d16e60f84db5d 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.common.requests;
 
+import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.network.NetworkSend;
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
@@ -76,7 +77,9 @@ public T build() {
 
     private final short version;
 
-    public AbstractRequest(short version) {
+    public AbstractRequest(ApiKeys api, short version) {
+        if (!api.isVersionSupported(version))
+            throw new UnsupportedVersionException("The " + api + " protocol does not support version " + version);
         this.version = version;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java
index 6fb9441ace649..2668ae1db5709 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java
@@ -86,7 +86,7 @@ public String toString() {
     private final String consumerGroupId;
 
     private AddOffsetsToTxnRequest(short version, String transactionalId, long producerId, short producerEpoch, String consumerGroupId) {
-        super(version);
+        super(ApiKeys.ADD_OFFSETS_TO_TXN, version);
         this.transactionalId = transactionalId;
         this.producerId = producerId;
         this.producerEpoch = producerEpoch;
@@ -94,7 +94,7 @@ private AddOffsetsToTxnRequest(short version, String transactionalId, long produ
     }
 
     public AddOffsetsToTxnRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.ADD_OFFSETS_TO_TXN, version);
         this.transactionalId = struct.get(TRANSACTIONAL_ID);
         this.producerId = struct.get(PRODUCER_ID);
         this.producerEpoch = struct.get(PRODUCER_EPOCH);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java
index 4a87289bbb079..3be7c9836abd7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java
@@ -102,7 +102,7 @@ public String toString() {
 
     private AddPartitionsToTxnRequest(short version, String transactionalId, long producerId, short producerEpoch,
                                       List partitions) {
-        super(version);
+        super(ApiKeys.ADD_PARTITIONS_TO_TXN, version);
         this.transactionalId = transactionalId;
         this.producerId = producerId;
         this.producerEpoch = producerEpoch;
@@ -110,7 +110,7 @@ private AddPartitionsToTxnRequest(short version, String transactionalId, long pr
     }
 
     public AddPartitionsToTxnRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.ADD_PARTITIONS_TO_TXN, version);
         this.transactionalId = struct.get(TRANSACTIONAL_ID);
         this.producerId = struct.get(PRODUCER_ID);
         this.producerEpoch = struct.get(PRODUCER_EPOCH);
@@ -150,7 +150,7 @@ protected Struct toStruct() {
         struct.set(PRODUCER_ID, producerId);
         struct.set(PRODUCER_EPOCH, producerEpoch);
 
-        Map> mappedPartitions = CollectionUtils.groupDataByTopic(partitions);
+        Map> mappedPartitions = CollectionUtils.groupPartitionsByTopic(partitions);
         Object[] partitionsArray = new Object[mappedPartitions.size()];
         int i = 0;
         for (Map.Entry> topicAndPartitions : mappedPartitions.entrySet()) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
index 977bd595b2724..ea8a073e01755 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
@@ -110,7 +110,7 @@ protected Struct toStruct(short version) {
         Struct struct = new Struct(ApiKeys.ADD_PARTITIONS_TO_TXN.responseSchema(version));
         struct.set(THROTTLE_TIME_MS, throttleTimeMs);
 
-        Map> errorsByTopic = CollectionUtils.groupDataByTopic(errors);
+        Map> errorsByTopic = CollectionUtils.groupPartitionDataByTopic(errors);
         List topics = new ArrayList<>(errorsByTopic.size());
         for (Map.Entry> entry : errorsByTopic.entrySet()) {
             Struct topicErrorCodes = struct.instance(ERRORS_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java
index ff1d0625c4dba..963ad065e4389 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java
@@ -123,13 +123,13 @@ public AlterConfigsRequest build(short version) {
     private final boolean validateOnly;
 
     public AlterConfigsRequest(short version, Map configs, boolean validateOnly) {
-        super(version);
+        super(ApiKeys.ALTER_CONFIGS, version);
         this.configs = Objects.requireNonNull(configs, "configs");
         this.validateOnly = validateOnly;
     }
 
     public AlterConfigsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.ALTER_CONFIGS, version);
         validateOnly = struct.getBoolean(VALIDATE_ONLY_KEY_NAME);
         Object[] resourcesArray = struct.getArray(RESOURCES_KEY_NAME);
         configs = new HashMap<>(resourcesArray.length);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java
index 0bc7cd0c61574..03bc0987eff00 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java
@@ -91,7 +91,7 @@ public String toString() {
     }
 
     public AlterReplicaLogDirsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.ALTER_REPLICA_LOG_DIRS, version);
         partitionDirs = new HashMap<>();
         for (Object logDirStructObj : struct.getArray(LOG_DIRS_KEY_NAME)) {
             Struct logDirStruct = (Struct) logDirStructObj;
@@ -108,7 +108,7 @@ public AlterReplicaLogDirsRequest(Struct struct, short version) {
     }
 
     public AlterReplicaLogDirsRequest(Map partitionDirs, short version) {
-        super(version);
+        super(ApiKeys.ALTER_REPLICA_LOG_DIRS, version);
         this.partitionDirs = partitionDirs;
     }
 
@@ -128,7 +128,7 @@ protected Struct toStruct() {
             logDirStruct.set(LOG_DIR_KEY_NAME, logDirEntry.getKey());
 
             List topicStructArray = new ArrayList<>();
-            for (Map.Entry> topicEntry: CollectionUtils.groupDataByTopic(logDirEntry.getValue()).entrySet()) {
+            for (Map.Entry> topicEntry: CollectionUtils.groupPartitionsByTopic(logDirEntry.getValue()).entrySet()) {
                 Struct topicStruct = logDirStruct.instance(TOPICS_KEY_NAME);
                 topicStruct.set(TOPIC_NAME, topicEntry.getKey());
                 topicStruct.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray());
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
index c8f6e4df20fc2..7a73275d74577 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
@@ -101,7 +101,7 @@ public AlterReplicaLogDirsResponse(int throttleTimeMs, Map> responsesByTopic = CollectionUtils.groupDataByTopic(responses);
+        Map> responsesByTopic = CollectionUtils.groupPartitionDataByTopic(responses);
         List topicStructArray = new ArrayList<>();
         for (Map.Entry> responsesByTopicEntry : responsesByTopic.entrySet()) {
             Struct topicStruct = struct.instance(TOPICS_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
index 347e3558fd2d8..e154ac92bb70c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
@@ -67,7 +67,7 @@ public ApiVersionsRequest(short version) {
     }
 
     public ApiVersionsRequest(short version, Short unsupportedRequestVersion) {
-        super(version);
+        super(ApiKeys.API_VERSIONS, version);
 
         // Unlike other request types, the broker handles ApiVersion requests with higher versions than
         // supported. It does so by treating the request as if it were v0 and returns a response using
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
index e6e873429c071..d6db1e1dc0a94 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
@@ -64,12 +64,12 @@ public String toString() {
     private final int brokerId;
 
     private ControlledShutdownRequest(int brokerId, short version) {
-        super(version);
+        super(ApiKeys.CONTROLLED_SHUTDOWN, version);
         this.brokerId = brokerId;
     }
 
     public ControlledShutdownRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.CONTROLLED_SHUTDOWN, version);
         brokerId = struct.getInt(BROKER_ID_KEY_NAME);
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java
index a77a373e29659..29ecd016f4b45 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java
@@ -123,14 +123,14 @@ public String toString() {
     private final List aclCreations;
 
     CreateAclsRequest(short version, List aclCreations) {
-        super(version);
+        super(ApiKeys.CREATE_ACLS, version);
         this.aclCreations = aclCreations;
 
         validate(aclCreations);
     }
 
     public CreateAclsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.CREATE_ACLS, version);
         this.aclCreations = new ArrayList<>();
         for (Object creationStructObj : struct.getArray(CREATIONS_KEY_NAME)) {
             Struct creationStruct = (Struct) creationStructObj;
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java
index 68e480faba85d..3277f100382e8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java
@@ -53,13 +53,13 @@ public class CreateDelegationTokenRequest extends AbstractRequest {
     private final long maxLifeTime;
 
     private CreateDelegationTokenRequest(short version, List renewers, long maxLifeTime) {
-        super(version);
+        super(ApiKeys.CREATE_DELEGATION_TOKEN, version);
         this.maxLifeTime = maxLifeTime;
         this.renewers = renewers;
     }
 
     public CreateDelegationTokenRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.CREATE_DELEGATION_TOKEN, version);
         maxLifeTime = struct.getLong(MAX_LIFE_TIME_KEY_NAME);
         Object[] renewerArray = struct.getArray(RENEWERS_KEY_NAME);
         renewers = new ArrayList<>();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java
index 5e776bb4dd37b..795a66a9ea670 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java
@@ -107,7 +107,7 @@ public String toString() {
     }
 
     CreatePartitionsRequest(Map newPartitions, int timeout, boolean validateOnly, short apiVersion) {
-        super(apiVersion);
+        super(ApiKeys.CREATE_PARTITIONS, apiVersion);
         this.newPartitions = newPartitions;
         this.duplicates = Collections.emptySet();
         this.timeout = timeout;
@@ -115,7 +115,7 @@ public String toString() {
     }
 
     public CreatePartitionsRequest(Struct struct, short apiVersion) {
-        super(apiVersion);
+        super(ApiKeys.CREATE_PARTITIONS, apiVersion);
         Object[] topicCountArray = struct.getArray(TOPIC_PARTITIONS_KEY_NAME);
         Map counts = new HashMap<>(topicCountArray.length);
         Set dupes = new HashSet<>();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
index aa346f5a260cc..9f05c5a5c0a58 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
@@ -197,7 +197,7 @@ public String toString() {
     public static final short NO_REPLICATION_FACTOR = -1;
 
     private CreateTopicsRequest(Map topics, Integer timeout, boolean validateOnly, short version) {
-        super(version);
+        super(ApiKeys.CREATE_TOPICS, version);
         this.topics = topics;
         this.timeout = timeout;
         this.validateOnly = validateOnly;
@@ -205,7 +205,7 @@ private CreateTopicsRequest(Map topics, Integer timeout, b
     }
 
     public CreateTopicsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.CREATE_TOPICS, version);
 
         Object[] requestStructs = struct.getArray(REQUESTS_KEY_NAME);
         Map topics = new HashMap<>();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java
index 4c19a4adbedeb..c3fc194cd378a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java
@@ -96,14 +96,14 @@ public String toString() {
     private final List filters;
 
     DeleteAclsRequest(short version, List filters) {
-        super(version);
+        super(ApiKeys.DELETE_ACLS, version);
         this.filters = filters;
 
         validate(version, filters);
     }
 
     public DeleteAclsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DELETE_ACLS, version);
         this.filters = new ArrayList<>();
         for (Object filterStructObj : struct.getArray(FILTERS)) {
             Struct filterStruct = (Struct) filterStructObj;
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
index 29604a505f8f6..f2a5d92e9882d 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
@@ -74,12 +74,12 @@ public String toString() {
     }
 
     private DeleteGroupsRequest(Set groups, short version) {
-        super(version);
+        super(ApiKeys.DELETE_GROUPS, version);
         this.groups = groups;
     }
 
     public DeleteGroupsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DELETE_GROUPS, version);
         Object[] groupsArray = struct.getArray(GROUPS_KEY_NAME);
         Set groups = new HashSet<>(groupsArray.length);
         for (Object group : groupsArray)
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java
index ad3db60788638..7ea55534f6908 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java
@@ -104,7 +104,7 @@ public String toString() {
 
 
     public DeleteRecordsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DELETE_RECORDS, version);
         partitionOffsets = new HashMap<>();
         for (Object topicStructObj : struct.getArray(TOPICS_KEY_NAME)) {
             Struct topicStruct = (Struct) topicStructObj;
@@ -120,14 +120,14 @@ public DeleteRecordsRequest(Struct struct, short version) {
     }
 
     public DeleteRecordsRequest(int timeout, Map partitionOffsets, short version) {
-        super(version);
+        super(ApiKeys.DELETE_RECORDS, version);
         this.timeout = timeout;
         this.partitionOffsets = partitionOffsets;
     }
     @Override
     protected Struct toStruct() {
         Struct struct = new Struct(ApiKeys.DELETE_RECORDS.requestSchema(version()));
-        Map> offsetsByTopic = CollectionUtils.groupDataByTopic(partitionOffsets);
+        Map> offsetsByTopic = CollectionUtils.groupPartitionDataByTopic(partitionOffsets);
         struct.set(TIMEOUT_KEY_NAME, timeout);
         List topicStructArray = new ArrayList<>();
         for (Map.Entry> offsetsByTopicEntry : offsetsByTopic.entrySet()) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java
index 0b494ba8c30cc..311be1f73e6ec 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java
@@ -136,7 +136,7 @@ public DeleteRecordsResponse(int throttleTimeMs, Map> responsesByTopic = CollectionUtils.groupDataByTopic(responses);
+        Map> responsesByTopic = CollectionUtils.groupPartitionDataByTopic(responses);
         List topicStructArray = new ArrayList<>();
         for (Map.Entry> responsesByTopicEntry : responsesByTopic.entrySet()) {
             Struct topicStruct = struct.instance(TOPICS_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
index 87f14b4aa6306..facb55e69f2a7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
@@ -92,13 +92,13 @@ public String toString() {
     }
 
     private DeleteTopicsRequest(Set topics, Integer timeout, short version) {
-        super(version);
+        super(ApiKeys.DELETE_TOPICS, version);
         this.topics = topics;
         this.timeout = timeout;
     }
 
     public DeleteTopicsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DELETE_TOPICS, version);
         Object[] topicsArray = struct.getArray(TOPICS_KEY_NAME);
         Set topics = new HashSet<>(topicsArray.length);
         for (Object topic : topicsArray)
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java
index d2198397f6daa..04bfee8d74057 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java
@@ -86,14 +86,14 @@ public String toString() {
     private final AclBindingFilter filter;
 
     DescribeAclsRequest(AclBindingFilter filter, short version) {
-        super(version);
+        super(ApiKeys.DELETE_ACLS, version);
         this.filter = filter;
 
         validate(filter, version);
     }
 
     public DescribeAclsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DELETE_ACLS, version);
         ResourcePatternFilter resourceFilter = RequestUtils.resourcePatternFilterFromStructFields(struct);
         AccessControlEntryFilter entryFilter = RequestUtils.aceFilterFromStructFields(struct);
         this.filter = new AclBindingFilter(resourceFilter, entryFilter);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java
index 781cd451b987e..0ee256ff53408 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java
@@ -100,13 +100,13 @@ public DescribeConfigsRequest build(short version) {
     private final boolean includeSynonyms;
 
     public DescribeConfigsRequest(short version, Map> resourceToConfigNames, boolean includeSynonyms) {
-        super(version);
+        super(ApiKeys.DESCRIBE_CONFIGS, version);
         this.resourceToConfigNames = Objects.requireNonNull(resourceToConfigNames, "resourceToConfigNames");
         this.includeSynonyms = includeSynonyms;
     }
 
     public DescribeConfigsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DESCRIBE_CONFIGS, version);
         Object[] resourcesArray = struct.getArray(RESOURCES_KEY_NAME);
         resourceToConfigNames = new HashMap<>(resourcesArray.length);
         for (Object resourceObj : resourcesArray) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java
index 574bbcc64603a..21e14608c3691 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java
@@ -69,12 +69,12 @@ public String toString() {
     }
 
     private DescribeDelegationTokenRequest(short version, List owners) {
-        super(version);
+        super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version);
         this.owners = owners;
     }
 
     public DescribeDelegationTokenRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, versionId);
 
         Object[] ownerArray = struct.getArray(OWNER_KEY_NAME);
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
index 8ea4a8c2176d2..006af4ff79cc4 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
@@ -72,12 +72,12 @@ public String toString() {
     private final List groupIds;
 
     private DescribeGroupsRequest(List groupIds, short version) {
-        super(version);
+        super(ApiKeys.DESCRIBE_GROUPS, version);
         this.groupIds = groupIds;
     }
 
     public DescribeGroupsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DESCRIBE_GROUPS, version);
         this.groupIds = new ArrayList<>();
         for (Object groupId : struct.getArray(GROUP_IDS_KEY_NAME))
             this.groupIds.add((String) groupId);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java
index 3728991040e82..e16cc1873328f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java
@@ -86,7 +86,7 @@ public String toString() {
     }
 
     public DescribeLogDirsRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.DESCRIBE_LOG_DIRS, version);
 
         if (struct.getArray(TOPICS_KEY_NAME) == null) {
             topicPartitions = null;
@@ -105,7 +105,7 @@ public DescribeLogDirsRequest(Struct struct, short version) {
 
     // topicPartitions == null indicates requesting all partitions, and an empty list indicates requesting no partitions.
     public DescribeLogDirsRequest(Set topicPartitions, short version) {
-        super(version);
+        super(ApiKeys.DESCRIBE_LOG_DIRS, version);
         this.topicPartitions = topicPartitions;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java
index 41c26173e3e5b..ee6d95cfff843 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java
@@ -137,7 +137,7 @@ protected Struct toStruct(short version) {
             logDirStruct.set(ERROR_CODE, logDirInfo.error.code());
             logDirStruct.set(LOG_DIR_KEY_NAME, logDirInfosEntry.getKey());
 
-            Map> replicaInfosByTopic = CollectionUtils.groupDataByTopic(logDirInfo.replicaInfos);
+            Map> replicaInfosByTopic = CollectionUtils.groupPartitionDataByTopic(logDirInfo.replicaInfos);
             List topicStructArray = new ArrayList<>();
             for (Map.Entry> replicaInfosByTopicEntry : replicaInfosByTopic.entrySet()) {
                 Struct topicStruct = logDirStruct.instance(TOPICS_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java
index 0ec22dc28313e..833a7fe5aedee 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java
@@ -89,7 +89,7 @@ public String toString() {
     private final TransactionResult result;
 
     private EndTxnRequest(short version, String transactionalId, long producerId, short producerEpoch, TransactionResult result) {
-        super(version);
+        super(ApiKeys.END_TXN, version);
         this.transactionalId = transactionalId;
         this.producerId = producerId;
         this.producerEpoch = producerEpoch;
@@ -97,7 +97,7 @@ private EndTxnRequest(short version, String transactionalId, long producerId, sh
     }
 
     public EndTxnRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.END_TXN, version);
         this.transactionalId = struct.get(TRANSACTIONAL_ID);
         this.producerId = struct.get(PRODUCER_ID);
         this.producerEpoch = struct.get(PRODUCER_EPOCH);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EpochEndOffset.java b/clients/src/main/java/org/apache/kafka/common/requests/EpochEndOffset.java
index ce938aad4f192..06dfef946d2bd 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/EpochEndOffset.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/EpochEndOffset.java
@@ -25,7 +25,6 @@
 /**
  * The offset, fetched from a leader, for a particular partition.
  */
-
 public class EpochEndOffset {
     public static final long UNDEFINED_EPOCH_OFFSET = NO_PARTITION_LEADER_EPOCH;
     public static final int UNDEFINED_EPOCH = NO_PARTITION_LEADER_EPOCH;
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java
index 21edb5e2f305f..5b996763cca9e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java
@@ -44,14 +44,14 @@ public class ExpireDelegationTokenRequest extends AbstractRequest {
     private static final Schema TOKEN_EXPIRE_REQUEST_V1 = TOKEN_EXPIRE_REQUEST_V0;
 
     private ExpireDelegationTokenRequest(short version, ByteBuffer hmac, long renewTimePeriod) {
-        super(version);
+        super(ApiKeys.EXPIRE_DELEGATION_TOKEN, version);
 
         this.hmac = hmac;
         this.expiryTimePeriod = renewTimePeriod;
     }
 
     public ExpireDelegationTokenRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.EXPIRE_DELEGATION_TOKEN, versionId);
 
         hmac = struct.getBytes(HMAC_KEY_NAME);
         expiryTimePeriod = struct.getLong(EXPIRY_TIME_PERIOD_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
index e013f5ef0d2b1..32eb24d7de2d1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -36,154 +35,166 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
+import static org.apache.kafka.common.protocol.CommonFields.CURRENT_LEADER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-import static org.apache.kafka.common.protocol.types.Type.INT8;
 import static org.apache.kafka.common.requests.FetchMetadata.FINAL_EPOCH;
 import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
 
 public class FetchRequest extends AbstractRequest {
     public static final int CONSUMER_REPLICA_ID = -1;
-    private static final String REPLICA_ID_KEY_NAME = "replica_id";
-    private static final String MAX_WAIT_KEY_NAME = "max_wait_time";
-    private static final String MIN_BYTES_KEY_NAME = "min_bytes";
-    private static final String ISOLATION_LEVEL_KEY_NAME = "isolation_level";
-    private static final String TOPICS_KEY_NAME = "topics";
-    private static final String FORGOTTEN_TOPICS_DATA = "forgotten_topics_data";
 
-    // request and partition level name
-    private static final String MAX_BYTES_KEY_NAME = "max_bytes";
-
-    // topic level field names
-    private static final String PARTITIONS_KEY_NAME = "partitions";
-
-    // partition level field names
-    private static final String FETCH_OFFSET_KEY_NAME = "fetch_offset";
-    private static final String LOG_START_OFFSET_KEY_NAME = "log_start_offset";
-
-    private static final Schema FETCH_REQUEST_PARTITION_V0 = new Schema(
-            PARTITION_ID,
-            new Field(FETCH_OFFSET_KEY_NAME, INT64, "Message offset."),
-            new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to fetch."));
-
-    // FETCH_REQUEST_PARTITION_V5 added log_start_offset field - the earliest available offset of partition data that can be consumed.
-    private static final Schema FETCH_REQUEST_PARTITION_V5 = new Schema(
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "Topics to fetch in the order provided.");
+    private static final Field.ComplexArray FORGOTTEN_TOPICS = new Field.ComplexArray("forgotten_topics_data",
+            "Topics to remove from the fetch session.");
+    private static final Field.Int32 MAX_BYTES = new Field.Int32("max_bytes",
+            "Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, " +
+                    "if the first message in the first non-empty partition of the fetch is larger than this " +
+                    "value, the message will still be returned to ensure that progress can be made.");
+    private static final Field.Int8 ISOLATION_LEVEL = new Field.Int8("isolation_level",
+            "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED " +
+                    "(isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), " +
+                    "non-transactional and COMMITTED transactional records are visible. To be more concrete, " +
+                    "READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), " +
+                    "and enables the inclusion of the list of aborted transactions in the result, which allows " +
+                    "consumers to discard ABORTED transactional records");
+    private static final Field.Int32 SESSION_ID = new Field.Int32("session_id", "The fetch session ID");
+    private static final Field.Int32 SESSION_EPOCH = new Field.Int32("session_epoch", "The fetch session epoch");
+
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "Partitions to fetch.");
+
+    // partition level fields
+    private static final Field.Int32 REPLICA_ID = new Field.Int32("replica_id",
+            "Broker id of the follower. For normal consumers, use -1.");
+    private static final Field.Int64 FETCH_OFFSET = new Field.Int64("fetch_offset", "Message offset.");
+    private static final Field.Int32 PARTITION_MAX_BYTES = new Field.Int32("partition_max_bytes",
+            "Maximum bytes to fetch.");
+    private static final Field.Int32 MAX_WAIT_TIME = new Field.Int32("max_wait_time",
+            "Maximum time in ms to wait for the response.");
+    private static final Field.Int32 MIN_BYTES = new Field.Int32("min_bytes",
+            "Minimum bytes to accumulate in the response.");
+    private static final Field.Int64 LOG_START_OFFSET = new Field.Int64("log_start_offset",
+            "Earliest available offset of the follower replica. " +
+                    "The field is only used when request is sent by follower. ");
+
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
-            new Field(FETCH_OFFSET_KEY_NAME, INT64, "Message offset."),
-            new Field(LOG_START_OFFSET_KEY_NAME, INT64, "Earliest available offset of the follower replica. " +
-                            "The field is only used when request is sent by follower. "),
-            new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to fetch."));
+            FETCH_OFFSET,
+            PARTITION_MAX_BYTES);
 
-    private static final Schema FETCH_REQUEST_TOPIC_V0 = new Schema(
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(FETCH_REQUEST_PARTITION_V0), "Partitions to fetch."));
-
-    private static final Schema FETCH_REQUEST_TOPIC_V5 = new Schema(
-            TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(FETCH_REQUEST_PARTITION_V5), "Partitions to fetch."));
+            PARTITIONS_V0);
 
     private static final Schema FETCH_REQUEST_V0 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(MAX_WAIT_KEY_NAME, INT32, "Maximum time in ms to wait for the response."),
-            new Field(MIN_BYTES_KEY_NAME, INT32, "Minimum bytes to accumulate in the response."),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(FETCH_REQUEST_TOPIC_V0), "Topics to fetch."));
+            REPLICA_ID,
+            MAX_WAIT_TIME,
+            MIN_BYTES,
+            TOPICS_V0);
 
     // The V1 Fetch Request body is the same as V0.
     // Only the version number is incremented to indicate a newer client
     private static final Schema FETCH_REQUEST_V1 = FETCH_REQUEST_V0;
-    // The V2 Fetch Request body is the same as V1.
-    // Only the version number is incremented to indicate the client support message format V1 which uses
-    // relative offset and has timestamp.
+
+    // V2 bumped to indicate the client support message format V1 which uses relative offset and has timestamp.
     private static final Schema FETCH_REQUEST_V2 = FETCH_REQUEST_V1;
-    // Fetch Request V3 added top level max_bytes field - the total size of partition data to accumulate in response.
+
+    // V3 added top level max_bytes field - the total size of partition data to accumulate in response.
     // The partition ordering is now relevant - partitions will be processed in order they appear in request.
     private static final Schema FETCH_REQUEST_V3 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(MAX_WAIT_KEY_NAME, INT32, "Maximum time in ms to wait for the response."),
-            new Field(MIN_BYTES_KEY_NAME, INT32, "Minimum bytes to accumulate in the response."),
-            new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, " +
-                    "if the first message in the first non-empty partition of the fetch is larger than this " +
-                    "value, the message will still be returned to ensure that progress can be made."),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(FETCH_REQUEST_TOPIC_V0), "Topics to fetch in the order provided."));
+            REPLICA_ID,
+            MAX_WAIT_TIME,
+            MIN_BYTES,
+            MAX_BYTES,
+            TOPICS_V0);
+
+    // V4 adds the fetch isolation level and exposes magic v2 (via the response).
 
-    // The V4 Fetch Request adds the fetch isolation level and exposes magic v2 (via the response).
     private static final Schema FETCH_REQUEST_V4 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(MAX_WAIT_KEY_NAME, INT32, "Maximum time in ms to wait for the response."),
-            new Field(MIN_BYTES_KEY_NAME, INT32, "Minimum bytes to accumulate in the response."),
-            new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, " +
-                    "if the first message in the first non-empty partition of the fetch is larger than this " +
-                    "value, the message will still be returned to ensure that progress can be made."),
-            new Field(ISOLATION_LEVEL_KEY_NAME, INT8, "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED " +
-                    "(isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), " +
-                    "non-transactional and COMMITTED transactional records are visible. To be more concrete, " +
-                    "READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), " +
-                    "and enables the inclusion of the list of aborted transactions in the result, which allows " +
-                    "consumers to discard ABORTED transactional records"),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(FETCH_REQUEST_TOPIC_V0), "Topics to fetch in the order provided."));
+            REPLICA_ID,
+            MAX_WAIT_TIME,
+            MIN_BYTES,
+            MAX_BYTES,
+            ISOLATION_LEVEL,
+            TOPICS_V0);
+
+
+    // V5 added log_start_offset field - the earliest available offset of partition data that can be consumed.
+    private static final Field PARTITIONS_V5 = PARTITIONS.withFields(
+            PARTITION_ID,
+            FETCH_OFFSET,
+            LOG_START_OFFSET,
+            PARTITION_MAX_BYTES);
+
+    private static final Field TOPICS_V5 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V5);
 
-    // FETCH_REQUEST_V5 added a per-partition log_start_offset field - the earliest available offset of partition data that can be consumed.
     private static final Schema FETCH_REQUEST_V5 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(MAX_WAIT_KEY_NAME, INT32, "Maximum time in ms to wait for the response."),
-            new Field(MIN_BYTES_KEY_NAME, INT32, "Minimum bytes to accumulate in the response."),
-            new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, " +
-                    "if the first message in the first non-empty partition of the fetch is larger than this " +
-                    "value, the message will still be returned to ensure that progress can be made."),
-            new Field(ISOLATION_LEVEL_KEY_NAME, INT8, "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED " +
-                    "(isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), " +
-                    "non-transactional and COMMITTED transactional records are visible. To be more concrete, " +
-                    "READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), " +
-                    "and enables the inclusion of the list of aborted transactions in the result, which allows " +
-                    "consumers to discard ABORTED transactional records"),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(FETCH_REQUEST_TOPIC_V5), "Topics to fetch in the order provided."));
-
-    /**
-     * The body of FETCH_REQUEST_V6 is the same as FETCH_REQUEST_V5.
-     * The version number is bumped up to indicate that the client supports KafkaStorageException.
-     * The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 5
-     */
+            REPLICA_ID,
+            MAX_WAIT_TIME,
+            MIN_BYTES,
+            MAX_BYTES,
+            ISOLATION_LEVEL,
+            TOPICS_V5);
+
+    // V6 bumped up to indicate that the client supports KafkaStorageException. The KafkaStorageException will be
+    // translated to NotLeaderForPartitionException in the response if version <= 5
     private static final Schema FETCH_REQUEST_V6 = FETCH_REQUEST_V5;
 
-    // FETCH_REQUEST_V7 added incremental fetch requests.
-    public static final Field.Int32 SESSION_ID = new Field.Int32("session_id", "The fetch session ID");
-    public static final Field.Int32 EPOCH = new Field.Int32("epoch", "The fetch epoch");
-
-    private static final Schema FORGOTTEN_TOPIC_DATA = new Schema(
-        TOPIC_NAME,
-        new Field(PARTITIONS_KEY_NAME, new ArrayOf(Type.INT32),
-            "Partitions to remove from the fetch session."));
+    // V7 added incremental fetch requests.
+    private static final Field.Array FORGOTTEN_PARTITIONS = new Field.Array("partitions", Type.INT32,
+            "Partitions to remove from the fetch session.");
+    private static final Field FORGOTTEN_TOPIC_DATA_V7 = FORGOTTEN_TOPICS.withFields(
+            TOPIC_NAME,
+            FORGOTTEN_PARTITIONS);
 
     private static final Schema FETCH_REQUEST_V7 = new Schema(
-        new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-        new Field(MAX_WAIT_KEY_NAME, INT32, "Maximum time in ms to wait for the response."),
-        new Field(MIN_BYTES_KEY_NAME, INT32, "Minimum bytes to accumulate in the response."),
-        new Field(MAX_BYTES_KEY_NAME, INT32, "Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, " +
-            "if the first message in the first non-empty partition of the fetch is larger than this " +
-            "value, the message will still be returned to ensure that progress can be made."),
-        new Field(ISOLATION_LEVEL_KEY_NAME, INT8, "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED " +
-            "(isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), " +
-            "non-transactional and COMMITTED transactional records are visible. To be more concrete, " +
-            "READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), " +
-            "and enables the inclusion of the list of aborted transactions in the result, which allows " +
-            "consumers to discard ABORTED transactional records"),
-        SESSION_ID,
-        EPOCH,
-        new Field(TOPICS_KEY_NAME, new ArrayOf(FETCH_REQUEST_TOPIC_V5), "Topics to fetch in the order provided."),
-        new Field(FORGOTTEN_TOPICS_DATA, new ArrayOf(FORGOTTEN_TOPIC_DATA), "Topics to remove from the fetch session."));
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+            REPLICA_ID,
+            MAX_WAIT_TIME,
+            MIN_BYTES,
+            MAX_BYTES,
+            ISOLATION_LEVEL,
+            SESSION_ID,
+            SESSION_EPOCH,
+            TOPICS_V5,
+            FORGOTTEN_TOPIC_DATA_V7);
+
+    // V8 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema FETCH_REQUEST_V8 = FETCH_REQUEST_V7;
 
+    // V9 adds the current leader epoch (see KIP-320)
+    private static final Field FETCH_REQUEST_PARTITION_V9 = PARTITIONS.withFields(
+            PARTITION_ID,
+            CURRENT_LEADER_EPOCH,
+            FETCH_OFFSET,
+            LOG_START_OFFSET,
+            PARTITION_MAX_BYTES);
+
+    private static final Field FETCH_REQUEST_TOPIC_V9 = TOPICS.withFields(
+            TOPIC_NAME,
+            FETCH_REQUEST_PARTITION_V9);
+
+    private static final Schema FETCH_REQUEST_V9 = new Schema(
+            REPLICA_ID,
+            MAX_WAIT_TIME,
+            MIN_BYTES,
+            MAX_BYTES,
+            ISOLATION_LEVEL,
+            SESSION_ID,
+            SESSION_EPOCH,
+            FETCH_REQUEST_TOPIC_V9,
+            FORGOTTEN_TOPIC_DATA_V7);
+
     public static Schema[] schemaVersions() {
         return new Schema[]{FETCH_REQUEST_V0, FETCH_REQUEST_V1, FETCH_REQUEST_V2, FETCH_REQUEST_V3, FETCH_REQUEST_V4,
-            FETCH_REQUEST_V5, FETCH_REQUEST_V6, FETCH_REQUEST_V7, FETCH_REQUEST_V8};
-    };
+            FETCH_REQUEST_V5, FETCH_REQUEST_V6, FETCH_REQUEST_V7, FETCH_REQUEST_V8, FETCH_REQUEST_V9};
+    }
 
     // default values for older versions where a request level limit did not exist
     public static final int DEFAULT_RESPONSE_MAX_BYTES = Integer.MAX_VALUE;
@@ -207,21 +218,27 @@ public static final class PartitionData {
         public final long fetchOffset;
         public final long logStartOffset;
         public final int maxBytes;
+        public final Optional currentLeaderEpoch;
 
-        public PartitionData(long fetchOffset, long logStartOffset, int maxBytes) {
+        public PartitionData(long fetchOffset, long logStartOffset, int maxBytes, Optional currentLeaderEpoch) {
             this.fetchOffset = fetchOffset;
             this.logStartOffset = logStartOffset;
             this.maxBytes = maxBytes;
+            this.currentLeaderEpoch = currentLeaderEpoch;
         }
 
         @Override
         public String toString() {
-            return "(offset=" + fetchOffset + ", logStartOffset=" + logStartOffset + ", maxBytes=" + maxBytes + ")";
+            return "(offset=" + fetchOffset +
+                    ", logStartOffset=" + logStartOffset +
+                    ", maxBytes=" + maxBytes +
+                    ", currentLeaderEpoch=" + currentLeaderEpoch +
+                    ")";
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(fetchOffset, logStartOffset, maxBytes);
+            return Objects.hash(fetchOffset, logStartOffset, maxBytes, currentLeaderEpoch);
         }
 
         @Override
@@ -231,7 +248,8 @@ public boolean equals(Object o) {
             PartitionData that = (PartitionData) o;
             return Objects.equals(fetchOffset, that.fetchOffset) &&
                 Objects.equals(logStartOffset, that.logStartOffset) &&
-                Objects.equals(maxBytes, that.maxBytes);
+                Objects.equals(maxBytes, that.maxBytes) &&
+                Objects.equals(currentLeaderEpoch, that.currentLeaderEpoch);
         }
     }
 
@@ -346,7 +364,7 @@ public String toString() {
     private FetchRequest(short version, int replicaId, int maxWait, int minBytes, int maxBytes,
                          Map fetchData, IsolationLevel isolationLevel,
                          List toForget, FetchMetadata metadata) {
-        super(version);
+        super(ApiKeys.FETCH, version);
         this.replicaId = replicaId;
         this.maxWait = maxWait;
         this.minBytes = minBytes;
@@ -358,44 +376,44 @@ private FetchRequest(short version, int replicaId, int maxWait, int minBytes, in
     }
 
     public FetchRequest(Struct struct, short version) {
-        super(version);
-        replicaId = struct.getInt(REPLICA_ID_KEY_NAME);
-        maxWait = struct.getInt(MAX_WAIT_KEY_NAME);
-        minBytes = struct.getInt(MIN_BYTES_KEY_NAME);
-        if (struct.hasField(MAX_BYTES_KEY_NAME))
-            maxBytes = struct.getInt(MAX_BYTES_KEY_NAME);
-        else
-            maxBytes = DEFAULT_RESPONSE_MAX_BYTES;
-        if (struct.hasField(ISOLATION_LEVEL_KEY_NAME))
-            isolationLevel = IsolationLevel.forId(struct.getByte(ISOLATION_LEVEL_KEY_NAME));
+        super(ApiKeys.FETCH, version);
+        replicaId = struct.get(REPLICA_ID);
+        maxWait = struct.get(MAX_WAIT_TIME);
+        minBytes = struct.get(MIN_BYTES);
+        maxBytes = struct.getOrElse(MAX_BYTES, DEFAULT_RESPONSE_MAX_BYTES);
+
+        if (struct.hasField(ISOLATION_LEVEL))
+            isolationLevel = IsolationLevel.forId(struct.get(ISOLATION_LEVEL));
         else
             isolationLevel = IsolationLevel.READ_UNCOMMITTED;
         toForget = new ArrayList<>(0);
-        if (struct.hasField(FORGOTTEN_TOPICS_DATA)) {
-            for (Object forgottenTopicObj : struct.getArray(FORGOTTEN_TOPICS_DATA)) {
+        if (struct.hasField(FORGOTTEN_TOPICS)) {
+            for (Object forgottenTopicObj : struct.get(FORGOTTEN_TOPICS)) {
                 Struct forgottenTopic = (Struct) forgottenTopicObj;
                 String topicName = forgottenTopic.get(TOPIC_NAME);
-                for (Object partObj : forgottenTopic.getArray(PARTITIONS_KEY_NAME)) {
+                for (Object partObj : forgottenTopic.get(FORGOTTEN_PARTITIONS)) {
                     Integer part = (Integer) partObj;
                     toForget.add(new TopicPartition(topicName, part));
                 }
             }
         }
         metadata = new FetchMetadata(struct.getOrElse(SESSION_ID, INVALID_SESSION_ID),
-            struct.getOrElse(EPOCH, FINAL_EPOCH));
+            struct.getOrElse(SESSION_EPOCH, FINAL_EPOCH));
 
         fetchData = new LinkedHashMap<>();
-        for (Object topicResponseObj : struct.getArray(TOPICS_KEY_NAME)) {
+        for (Object topicResponseObj : struct.get(TOPICS)) {
             Struct topicResponse = (Struct) topicResponseObj;
             String topic = topicResponse.get(TOPIC_NAME);
-            for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
                 Struct partitionResponse = (Struct) partitionResponseObj;
                 int partition = partitionResponse.get(PARTITION_ID);
-                long offset = partitionResponse.getLong(FETCH_OFFSET_KEY_NAME);
-                int maxBytes = partitionResponse.getInt(MAX_BYTES_KEY_NAME);
-                long logStartOffset = partitionResponse.hasField(LOG_START_OFFSET_KEY_NAME) ?
-                    partitionResponse.getLong(LOG_START_OFFSET_KEY_NAME) : INVALID_LOG_START_OFFSET;
-                PartitionData partitionData = new PartitionData(offset, logStartOffset, maxBytes);
+                long offset = partitionResponse.get(FETCH_OFFSET);
+                int maxBytes = partitionResponse.get(PARTITION_MAX_BYTES);
+                long logStartOffset = partitionResponse.getOrElse(LOG_START_OFFSET, INVALID_LOG_START_OFFSET);
+
+                // Current leader epoch added in v9
+                Optional currentLeaderEpoch = RequestUtils.getLeaderEpoch(partitionResponse, CURRENT_LEADER_EPOCH);
+                PartitionData partitionData = new PartitionData(offset, logStartOffset, maxBytes, currentLeaderEpoch);
                 fetchData.put(new TopicPartition(topic, partition), partitionData);
             }
         }
@@ -410,14 +428,14 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         // may not be any partitions at all in the response.  For this reason, the top-level error code
         // is essential for them.
         Errors error = Errors.forException(e);
-        LinkedHashMap responseData = new LinkedHashMap<>();
+        LinkedHashMap> responseData = new LinkedHashMap<>();
         for (Map.Entry entry : fetchData.entrySet()) {
-            FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData(error,
+            FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData<>(error,
                 FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET,
                 FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY);
             responseData.put(entry.getKey(), partitionResponse);
         }
-        return new FetchResponse(error, responseData, throttleTimeMs, metadata.sessionId());
+        return new FetchResponse<>(error, responseData, throttleTimeMs, metadata.sessionId());
     }
 
     public int replicaId() {
@@ -466,53 +484,47 @@ protected Struct toStruct() {
         List> topicsData =
             TopicAndPartitionData.batchByTopic(fetchData.entrySet().iterator());
 
-        struct.set(REPLICA_ID_KEY_NAME, replicaId);
-        struct.set(MAX_WAIT_KEY_NAME, maxWait);
-        struct.set(MIN_BYTES_KEY_NAME, minBytes);
-        if (struct.hasField(MAX_BYTES_KEY_NAME))
-            struct.set(MAX_BYTES_KEY_NAME, maxBytes);
-        if (struct.hasField(ISOLATION_LEVEL_KEY_NAME))
-            struct.set(ISOLATION_LEVEL_KEY_NAME, isolationLevel.id());
+        struct.set(REPLICA_ID, replicaId);
+        struct.set(MAX_WAIT_TIME, maxWait);
+        struct.set(MIN_BYTES, minBytes);
+        struct.setIfExists(MAX_BYTES, maxBytes);
+        struct.setIfExists(ISOLATION_LEVEL, isolationLevel.id());
         struct.setIfExists(SESSION_ID, metadata.sessionId());
-        struct.setIfExists(EPOCH, metadata.epoch());
+        struct.setIfExists(SESSION_EPOCH, metadata.epoch());
 
         List topicArray = new ArrayList<>();
         for (TopicAndPartitionData topicEntry : topicsData) {
-            Struct topicData = struct.instance(TOPICS_KEY_NAME);
+            Struct topicData = struct.instance(TOPICS);
             topicData.set(TOPIC_NAME, topicEntry.topic);
             List partitionArray = new ArrayList<>();
             for (Map.Entry partitionEntry : topicEntry.partitions.entrySet()) {
                 PartitionData fetchPartitionData = partitionEntry.getValue();
-                Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+                Struct partitionData = topicData.instance(PARTITIONS);
                 partitionData.set(PARTITION_ID, partitionEntry.getKey());
-                partitionData.set(FETCH_OFFSET_KEY_NAME, fetchPartitionData.fetchOffset);
-                if (partitionData.hasField(LOG_START_OFFSET_KEY_NAME))
-                    partitionData.set(LOG_START_OFFSET_KEY_NAME, fetchPartitionData.logStartOffset);
-                partitionData.set(MAX_BYTES_KEY_NAME, fetchPartitionData.maxBytes);
+                partitionData.set(FETCH_OFFSET, fetchPartitionData.fetchOffset);
+                partitionData.set(PARTITION_MAX_BYTES, fetchPartitionData.maxBytes);
+                partitionData.setIfExists(LOG_START_OFFSET, fetchPartitionData.logStartOffset);
+                RequestUtils.setLeaderEpochIfExists(partitionData, CURRENT_LEADER_EPOCH, fetchPartitionData.currentLeaderEpoch);
                 partitionArray.add(partitionData);
             }
-            topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+            topicData.set(PARTITIONS, partitionArray.toArray());
             topicArray.add(topicData);
         }
-        struct.set(TOPICS_KEY_NAME, topicArray.toArray());
-        if (struct.hasField(FORGOTTEN_TOPICS_DATA)) {
+        struct.set(TOPICS, topicArray.toArray());
+        if (struct.hasField(FORGOTTEN_TOPICS)) {
             Map> topicsToPartitions = new HashMap<>();
             for (TopicPartition part : toForget) {
-                List partitions = topicsToPartitions.get(part.topic());
-                if (partitions == null) {
-                    partitions = new ArrayList<>();
-                    topicsToPartitions.put(part.topic(), partitions);
-                }
+                List partitions = topicsToPartitions.computeIfAbsent(part.topic(), topic -> new ArrayList<>());
                 partitions.add(part.partition());
             }
             List toForgetStructs = new ArrayList<>();
             for (Map.Entry> entry : topicsToPartitions.entrySet()) {
-                Struct toForgetStruct = struct.instance(FORGOTTEN_TOPICS_DATA);
+                Struct toForgetStruct = struct.instance(FORGOTTEN_TOPICS);
                 toForgetStruct.set(TOPIC_NAME, entry.getKey());
-                toForgetStruct.set(PARTITIONS_KEY_NAME, entry.getValue().toArray());
+                toForgetStruct.set(FORGOTTEN_PARTITIONS, entry.getValue().toArray());
                 toForgetStructs.add(toForgetStruct);
             }
-            struct.set(FORGOTTEN_TOPICS_DATA, toForgetStructs.toArray());
+            struct.set(FORGOTTEN_TOPICS, toForgetStructs.toArray());
         }
         return struct;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
index 16e33965e9ec2..2e0eaf2fc2813 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.network.ByteBufferSend;
-import org.apache.kafka.common.record.MultiRecordsSend;
 import org.apache.kafka.common.network.Send;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
@@ -28,6 +27,7 @@
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.record.BaseRecords;
 import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.record.MultiRecordsSend;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayDeque;
@@ -43,13 +43,20 @@
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
 import static org.apache.kafka.common.protocol.types.Type.RECORDS;
 import static org.apache.kafka.common.protocol.types.Type.STRING;
 import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
 
 /**
  * This wrapper supports all versions of the Fetch API
+ *
+ * Possible error codes:
+ *
+ *  OFFSET_OUT_OF_RANGE (1)
+ *  UNKNOWN_TOPIC_OR_PARTITION (3)
+ *  NOT_LEADER_FOR_PARTITION (6)
+ *  REPLICA_NOT_AVAILABLE (9)
+ *  UNKNOWN (-1)
  */
 public class FetchResponse extends AbstractResponse {
 
@@ -58,22 +65,20 @@ public class FetchResponse extends AbstractResponse {
     // topic level field names
     private static final String PARTITIONS_KEY_NAME = "partition_responses";
 
-    // partition level field names
+    // partition level fields
+    private static final Field.Int64 HIGH_WATERMARK = new Field.Int64("high_watermark",
+            "Last committed offset.");
+    private static final Field.Int64 LOG_START_OFFSET = new Field.Int64("log_start_offset",
+            "Earliest available offset.");
+
     private static final String PARTITION_HEADER_KEY_NAME = "partition_header";
-    private static final String HIGH_WATERMARK_KEY_NAME = "high_watermark";
-    private static final String LAST_STABLE_OFFSET_KEY_NAME = "last_stable_offset";
-    private static final String LOG_START_OFFSET_KEY_NAME = "log_start_offset";
     private static final String ABORTED_TRANSACTIONS_KEY_NAME = "aborted_transactions";
     private static final String RECORD_SET_KEY_NAME = "record_set";
 
-    // aborted transaction field names
-    private static final String PRODUCER_ID_KEY_NAME = "producer_id";
-    private static final String FIRST_OFFSET_KEY_NAME = "first_offset";
-
     private static final Schema FETCH_RESPONSE_PARTITION_HEADER_V0 = new Schema(
             PARTITION_ID,
             ERROR_CODE,
-            new Field(HIGH_WATERMARK_KEY_NAME, INT64, "Last committed offset."));
+            HIGH_WATERMARK);
     private static final Schema FETCH_RESPONSE_PARTITION_V0 = new Schema(
             new Field(PARTITION_HEADER_KEY_NAME, FETCH_RESPONSE_PARTITION_HEADER_V0),
             new Field(RECORD_SET_KEY_NAME, RECORDS));
@@ -85,42 +90,47 @@ public class FetchResponse extends AbstractResponse {
     private static final Schema FETCH_RESPONSE_V0 = new Schema(
             new Field(RESPONSES_KEY_NAME, new ArrayOf(FETCH_RESPONSE_TOPIC_V0)));
 
+    // V1 bumped for the addition of the throttle time
     private static final Schema FETCH_RESPONSE_V1 = new Schema(
             THROTTLE_TIME_MS,
             new Field(RESPONSES_KEY_NAME, new ArrayOf(FETCH_RESPONSE_TOPIC_V0)));
-    // Even though fetch response v2 has the same protocol as v1, the record set in the response is different. In v1,
-    // record set only includes messages of v0 (magic byte 0). In v2, record set can include messages of v0 and v1
-    // (magic byte 0 and 1). For details, see Records, RecordBatch and Record.
+
+    // V2 bumped to indicate the client support message format V1 which uses relative offset and has timestamp.
     private static final Schema FETCH_RESPONSE_V2 = FETCH_RESPONSE_V1;
 
-    // The partition ordering is now relevant - partitions will be processed in order they appear in request.
+    // V3 bumped for addition of top-levl max_bytes field and to indicate that partition ordering is relevant
     private static final Schema FETCH_RESPONSE_V3 = FETCH_RESPONSE_V2;
 
-    // The v4 Fetch Response adds features for transactional consumption (the aborted transaction list and the
+    // V4 adds features for transactional consumption (the aborted transaction list and the
     // last stable offset). It also exposes messages with magic v2 (along with older formats).
-    private static final Schema FETCH_RESPONSE_ABORTED_TRANSACTION_V4 = new Schema(
-            new Field(PRODUCER_ID_KEY_NAME, INT64, "The producer id associated with the aborted transactions"),
-            new Field(FIRST_OFFSET_KEY_NAME, INT64, "The first offset in the aborted transaction"));
+    // aborted transaction field names
+    private static final Field.Int64 LAST_STABLE_OFFSET = new Field.Int64("last_stable_offset",
+            "The last stable offset (or LSO) of the partition. This is the last offset such that the state " +
+                    "of all transactional records prior to this offset have been decided (ABORTED or COMMITTED)");
+    private static final Field.Int64 PRODUCER_ID = new Field.Int64("producer_id",
+            "The producer id associated with the aborted transactions");
+    private static final Field.Int64 FIRST_OFFSET = new Field.Int64("first_offset",
+            "The first offset in the aborted transaction");
 
-    private static final Schema FETCH_RESPONSE_ABORTED_TRANSACTION_V5 = FETCH_RESPONSE_ABORTED_TRANSACTION_V4;
+    private static final Schema FETCH_RESPONSE_ABORTED_TRANSACTION_V4 = new Schema(
+            PRODUCER_ID,
+            FIRST_OFFSET);
 
     private static final Schema FETCH_RESPONSE_PARTITION_HEADER_V4 = new Schema(
             PARTITION_ID,
             ERROR_CODE,
-            new Field(HIGH_WATERMARK_KEY_NAME, INT64, "Last committed offset."),
-            new Field(LAST_STABLE_OFFSET_KEY_NAME, INT64, "The last stable offset (or LSO) of the partition. This is the last offset such that the state " +
-                    "of all transactional records prior to this offset have been decided (ABORTED or COMMITTED)"),
+            HIGH_WATERMARK,
+            LAST_STABLE_OFFSET,
             new Field(ABORTED_TRANSACTIONS_KEY_NAME, ArrayOf.nullable(FETCH_RESPONSE_ABORTED_TRANSACTION_V4)));
 
-    // FETCH_RESPONSE_PARTITION_HEADER_V5 added log_start_offset field - the earliest available offset of partition data that can be consumed.
+    // V5 added log_start_offset field - the earliest available offset of partition data that can be consumed.
     private static final Schema FETCH_RESPONSE_PARTITION_HEADER_V5 = new Schema(
             PARTITION_ID,
             ERROR_CODE,
-            new Field(HIGH_WATERMARK_KEY_NAME, INT64, "Last committed offset."),
-            new Field(LAST_STABLE_OFFSET_KEY_NAME, INT64, "The last stable offset (or LSO) of the partition. This is the last offset such that the state " +
-                    "of all transactional records prior to this offset have been decided (ABORTED or COMMITTED)"),
-            new Field(LOG_START_OFFSET_KEY_NAME, INT64, "Earliest available offset."),
-            new Field(ABORTED_TRANSACTIONS_KEY_NAME, ArrayOf.nullable(FETCH_RESPONSE_ABORTED_TRANSACTION_V5)));
+            HIGH_WATERMARK,
+            LAST_STABLE_OFFSET,
+            LOG_START_OFFSET,
+            new Field(ABORTED_TRANSACTIONS_KEY_NAME, ArrayOf.nullable(FETCH_RESPONSE_ABORTED_TRANSACTION_V4)));
 
     private static final Schema FETCH_RESPONSE_PARTITION_V4 = new Schema(
             new Field(PARTITION_HEADER_KEY_NAME, FETCH_RESPONSE_PARTITION_HEADER_V4),
@@ -146,15 +156,12 @@ public class FetchResponse extends AbstractResponse {
             THROTTLE_TIME_MS,
             new Field(RESPONSES_KEY_NAME, new ArrayOf(FETCH_RESPONSE_TOPIC_V5)));
 
-    /**
-     * The body of FETCH_RESPONSE_V6 is the same as FETCH_RESPONSE_V5.
-     * The version number is bumped up to indicate that the client supports KafkaStorageException.
-     * The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 5
-     */
+    // V6 bumped up to indicate that the client supports KafkaStorageException. The KafkaStorageException will
+    // be translated to NotLeaderForPartitionException in the response if version <= 5
     private static final Schema FETCH_RESPONSE_V6 = FETCH_RESPONSE_V5;
 
-    // FETCH_RESPONSE_V7 added incremental fetch responses and a top-level error code.
-    public static final Field.Int32 SESSION_ID = new Field.Int32("session_id", "The fetch session ID");
+    // V7 added incremental fetch responses and a top-level error code.
+    private static final Field.Int32 SESSION_ID = new Field.Int32("session_id", "The fetch session ID");
 
     private static final Schema FETCH_RESPONSE_V7 = new Schema(
             THROTTLE_TIME_MS,
@@ -162,32 +169,22 @@ public class FetchResponse extends AbstractResponse {
             SESSION_ID,
             new Field(RESPONSES_KEY_NAME, new ArrayOf(FETCH_RESPONSE_TOPIC_V5)));
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V8 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema FETCH_RESPONSE_V8 = FETCH_RESPONSE_V7;
 
+    // V9 adds the current leader epoch (see KIP-320)
+    private static final Schema FETCH_RESPONSE_V9 = FETCH_RESPONSE_V8;
+
     public static Schema[] schemaVersions() {
         return new Schema[] {FETCH_RESPONSE_V0, FETCH_RESPONSE_V1, FETCH_RESPONSE_V2,
             FETCH_RESPONSE_V3, FETCH_RESPONSE_V4, FETCH_RESPONSE_V5, FETCH_RESPONSE_V6,
-            FETCH_RESPONSE_V7, FETCH_RESPONSE_V8};
+            FETCH_RESPONSE_V7, FETCH_RESPONSE_V8, FETCH_RESPONSE_V9};
     }
 
-
     public static final long INVALID_HIGHWATERMARK = -1L;
     public static final long INVALID_LAST_STABLE_OFFSET = -1L;
     public static final long INVALID_LOG_START_OFFSET = -1L;
 
-    /**
-     * Possible error codes:
-     *
-     *  OFFSET_OUT_OF_RANGE (1)
-     *  UNKNOWN_TOPIC_OR_PARTITION (3)
-     *  NOT_LEADER_FOR_PARTITION (6)
-     *  REPLICA_NOT_AVAILABLE (9)
-     *  UNKNOWN (-1)
-     */
-
     private final int throttleTimeMs;
     private final Errors error;
     private final int sessionId;
@@ -318,13 +315,9 @@ public static FetchResponse parse(Struct struct) {
                 Struct partitionResponseHeader = partitionResponse.getStruct(PARTITION_HEADER_KEY_NAME);
                 int partition = partitionResponseHeader.get(PARTITION_ID);
                 Errors error = Errors.forCode(partitionResponseHeader.get(ERROR_CODE));
-                long highWatermark = partitionResponseHeader.getLong(HIGH_WATERMARK_KEY_NAME);
-                long lastStableOffset = INVALID_LAST_STABLE_OFFSET;
-                if (partitionResponseHeader.hasField(LAST_STABLE_OFFSET_KEY_NAME))
-                    lastStableOffset = partitionResponseHeader.getLong(LAST_STABLE_OFFSET_KEY_NAME);
-                long logStartOffset = INVALID_LOG_START_OFFSET;
-                if (partitionResponseHeader.hasField(LOG_START_OFFSET_KEY_NAME))
-                    logStartOffset = partitionResponseHeader.getLong(LOG_START_OFFSET_KEY_NAME);
+                long highWatermark = partitionResponseHeader.get(HIGH_WATERMARK);
+                long lastStableOffset = partitionResponseHeader.getOrElse(LAST_STABLE_OFFSET, INVALID_LAST_STABLE_OFFSET);
+                long logStartOffset = partitionResponseHeader.getOrElse(LOG_START_OFFSET, INVALID_LOG_START_OFFSET);
 
                 BaseRecords baseRecords = partitionResponse.getRecords(RECORD_SET_KEY_NAME);
                 if (!(baseRecords instanceof MemoryRecords))
@@ -338,8 +331,8 @@ public static FetchResponse parse(Struct struct) {
                         abortedTransactions = new ArrayList<>(abortedTransactionsArray.length);
                         for (Object abortedTransactionObj : abortedTransactionsArray) {
                             Struct abortedTransactionStruct = (Struct) abortedTransactionObj;
-                            long producerId = abortedTransactionStruct.getLong(PRODUCER_ID_KEY_NAME);
-                            long firstOffset = abortedTransactionStruct.getLong(FIRST_OFFSET_KEY_NAME);
+                            long producerId = abortedTransactionStruct.get(PRODUCER_ID);
+                            long firstOffset = abortedTransactionStruct.get(FIRST_OFFSET);
                             abortedTransactions.add(new AbortedTransaction(producerId, firstOffset));
                         }
                     }
@@ -490,10 +483,10 @@ private static  Struct toStruct(short version, int thrott
                 Struct partitionDataHeader = partitionData.instance(PARTITION_HEADER_KEY_NAME);
                 partitionDataHeader.set(PARTITION_ID, partitionEntry.getKey());
                 partitionDataHeader.set(ERROR_CODE, errorCode);
-                partitionDataHeader.set(HIGH_WATERMARK_KEY_NAME, fetchPartitionData.highWatermark);
+                partitionDataHeader.set(HIGH_WATERMARK, fetchPartitionData.highWatermark);
 
-                if (partitionDataHeader.hasField(LAST_STABLE_OFFSET_KEY_NAME)) {
-                    partitionDataHeader.set(LAST_STABLE_OFFSET_KEY_NAME, fetchPartitionData.lastStableOffset);
+                if (partitionDataHeader.hasField(LAST_STABLE_OFFSET)) {
+                    partitionDataHeader.set(LAST_STABLE_OFFSET, fetchPartitionData.lastStableOffset);
 
                     if (fetchPartitionData.abortedTransactions == null) {
                         partitionDataHeader.set(ABORTED_TRANSACTIONS_KEY_NAME, null);
@@ -501,16 +494,14 @@ private static  Struct toStruct(short version, int thrott
                         List abortedTransactionStructs = new ArrayList<>(fetchPartitionData.abortedTransactions.size());
                         for (AbortedTransaction abortedTransaction : fetchPartitionData.abortedTransactions) {
                             Struct abortedTransactionStruct = partitionDataHeader.instance(ABORTED_TRANSACTIONS_KEY_NAME);
-                            abortedTransactionStruct.set(PRODUCER_ID_KEY_NAME, abortedTransaction.producerId);
-                            abortedTransactionStruct.set(FIRST_OFFSET_KEY_NAME, abortedTransaction.firstOffset);
+                            abortedTransactionStruct.set(PRODUCER_ID, abortedTransaction.producerId);
+                            abortedTransactionStruct.set(FIRST_OFFSET, abortedTransaction.firstOffset);
                             abortedTransactionStructs.add(abortedTransactionStruct);
                         }
                         partitionDataHeader.set(ABORTED_TRANSACTIONS_KEY_NAME, abortedTransactionStructs.toArray());
                     }
                 }
-                if (partitionDataHeader.hasField(LOG_START_OFFSET_KEY_NAME))
-                    partitionDataHeader.set(LOG_START_OFFSET_KEY_NAME, fetchPartitionData.logStartOffset);
-
+                partitionDataHeader.setIfExists(LOG_START_OFFSET, fetchPartitionData.logStartOffset);
                 partitionData.set(PARTITION_HEADER_KEY_NAME, partitionDataHeader);
                 partitionData.set(RECORD_SET_KEY_NAME, fetchPartitionData.records);
                 partitionArray.add(partitionData);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java
index 8fb71a87c07e3..2d44ab3015bae 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java
@@ -94,13 +94,13 @@ public String toString() {
     private final CoordinatorType coordinatorType;
 
     private FindCoordinatorRequest(CoordinatorType coordinatorType, String coordinatorKey, short version) {
-        super(version);
+        super(ApiKeys.FIND_COORDINATOR, version);
         this.coordinatorType = coordinatorType;
         this.coordinatorKey = coordinatorKey;
     }
 
     public FindCoordinatorRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.FIND_COORDINATOR, version);
 
         if (struct.hasField(COORDINATOR_TYPE_KEY_NAME))
             this.coordinatorType = CoordinatorType.forId(struct.getByte(COORDINATOR_TYPE_KEY_NAME));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
index 5a70ed87dcd46..9a131478df723 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
@@ -80,14 +80,14 @@ public String toString() {
     private final String memberId;
 
     private HeartbeatRequest(String groupId, int groupGenerationId, String memberId, short version) {
-        super(version);
+        super(ApiKeys.HEARTBEAT, version);
         this.groupId = groupId;
         this.groupGenerationId = groupGenerationId;
         this.memberId = memberId;
     }
 
     public HeartbeatRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.HEARTBEAT, version);
         groupId = struct.get(GROUP_ID);
         groupGenerationId = struct.get(GENERATION_ID);
         memberId = struct.get(MEMBER_ID);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java
index c350599710269..aab7c72cf430e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java
@@ -82,13 +82,13 @@ public String toString() {
     }
 
     public InitProducerIdRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.INIT_PRODUCER_ID, version);
         this.transactionalId = struct.get(NULLABLE_TRANSACTIONAL_ID);
         this.transactionTimeoutMs = struct.getInt(TRANSACTION_TIMEOUT_KEY_NAME);
     }
 
     private InitProducerIdRequest(short version, String transactionalId, int transactionTimeoutMs) {
-        super(version);
+        super(ApiKeys.INIT_PRODUCER_ID, version);
         this.transactionalId = transactionalId;
         this.transactionTimeoutMs = transactionTimeoutMs;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
index ba009a1ae2b5d..366509dbab1f7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
@@ -160,7 +160,7 @@ public String toString() {
     private JoinGroupRequest(short version, String groupId, int sessionTimeout,
             int rebalanceTimeout, String memberId, String protocolType,
             List groupProtocols) {
-        super(version);
+        super(ApiKeys.JOIN_GROUP, version);
         this.groupId = groupId;
         this.sessionTimeout = sessionTimeout;
         this.rebalanceTimeout = rebalanceTimeout;
@@ -170,7 +170,7 @@ private JoinGroupRequest(short version, String groupId, int sessionTimeout,
     }
 
     public JoinGroupRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.JOIN_GROUP, versionId);
 
         groupId = struct.get(GROUP_ID);
         sessionTimeout = struct.getInt(SESSION_TIMEOUT_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 4fa5337188489..b821c0e948f79 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -145,7 +145,7 @@ public String toString() {
 
     private LeaderAndIsrRequest(int controllerId, int controllerEpoch, Map partitionStates,
                                 Set liveLeaders, short version) {
-        super(version);
+        super(ApiKeys.LEADER_AND_ISR, version);
         this.controllerId = controllerId;
         this.controllerEpoch = controllerEpoch;
         this.partitionStates = partitionStates;
@@ -153,7 +153,7 @@ private LeaderAndIsrRequest(int controllerId, int controllerEpoch, Map partitionStates = new HashMap<>();
         for (Object partitionStateDataObj : struct.getArray(PARTITION_STATES_KEY_NAME)) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
index 2b4acf876ff9a..2d0b448262f94 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
@@ -74,13 +74,13 @@ public String toString() {
     private final String memberId;
 
     private LeaveGroupRequest(String groupId, String memberId, short version) {
-        super(version);
+        super(ApiKeys.LEAVE_GROUP, version);
         this.groupId = groupId;
         this.memberId = memberId;
     }
 
     public LeaveGroupRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.LEAVE_GROUP, version);
         groupId = struct.get(GROUP_ID);
         memberId = struct.get(MEMBER_ID);
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
index f64f71a56e37e..27254cb1f2404 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
@@ -59,11 +59,11 @@ public String toString() {
     }
 
     public ListGroupsRequest(short version) {
-        super(version);
+        super(ApiKeys.LIST_GROUPS, version);
     }
 
     public ListGroupsRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.LIST_GROUPS, versionId);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
index be094feec5587..5107c4e9140d4 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -32,13 +31,12 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
+import static org.apache.kafka.common.protocol.CommonFields.CURRENT_LEADER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-import static org.apache.kafka.common.protocol.types.Type.INT8;
 
 public class ListOffsetRequest extends AbstractRequest {
     public static final long EARLIEST_TIMESTAMP = -2L;
@@ -47,70 +45,93 @@ public class ListOffsetRequest extends AbstractRequest {
     public static final int CONSUMER_REPLICA_ID = -1;
     public static final int DEBUGGING_REPLICA_ID = -2;
 
-    private static final String REPLICA_ID_KEY_NAME = "replica_id";
-    private static final String ISOLATION_LEVEL_KEY_NAME = "isolation_level";
-    private static final String TOPICS_KEY_NAME = "topics";
+    // top level fields
+    private static final Field.Int32 REPLICA_ID = new Field.Int32("replica_id",
+            "Broker id of the follower. For normal consumers, use -1.");
+    private static final Field.Int8 ISOLATION_LEVEL = new Field.Int8("isolation_level",
+            "This setting controls the visibility of transactional records. " +
+                    "Using READ_UNCOMMITTED (isolation_level = 0) makes all records visible. With READ_COMMITTED " +
+                    "(isolation_level = 1), non-transactional and COMMITTED transactional records are visible. " +
+                    "To be more concrete, READ_COMMITTED returns all data from offsets smaller than the current " +
+                    "LSO (last stable offset), and enables the inclusion of the list of aborted transactions in the " +
+                    "result, which allows consumers to discard ABORTED transactional records");
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "Topics to list offsets.");
 
-    // topic level field names
-    private static final String PARTITIONS_KEY_NAME = "partitions";
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "Partitions to list offsets.");
 
-    // partition level field names
-    private static final String TIMESTAMP_KEY_NAME = "timestamp";
-    private static final String MAX_NUM_OFFSETS_KEY_NAME = "max_num_offsets";
+    // partition level fields
+    private static final Field.Int64 TIMESTAMP = new Field.Int64("timestamp",
+            "The target timestamp for the partition.");
+    private static final Field.Int32 MAX_NUM_OFFSETS = new Field.Int32("max_num_offsets",
+            "Maximum offsets to return.");
 
-    private static final Schema LIST_OFFSET_REQUEST_PARTITION_V0 = new Schema(
-            PARTITION_ID,
-            new Field(TIMESTAMP_KEY_NAME, INT64, "Timestamp."),
-            new Field(MAX_NUM_OFFSETS_KEY_NAME, INT32, "Maximum offsets to return."));
-    private static final Schema LIST_OFFSET_REQUEST_PARTITION_V1 = new Schema(
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
-            new Field(TIMESTAMP_KEY_NAME, INT64, "The target timestamp for the partition."));
+            TIMESTAMP,
+            MAX_NUM_OFFSETS);
 
-    private static final Schema LIST_OFFSET_REQUEST_TOPIC_V0 = new Schema(
-            TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(LIST_OFFSET_REQUEST_PARTITION_V0), "Partitions to list offset."));
-    private static final Schema LIST_OFFSET_REQUEST_TOPIC_V1 = new Schema(
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(LIST_OFFSET_REQUEST_PARTITION_V1), "Partitions to list offset."));
+            PARTITIONS_V0);
 
     private static final Schema LIST_OFFSET_REQUEST_V0 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(LIST_OFFSET_REQUEST_TOPIC_V0), "Topics to list offsets."));
+            REPLICA_ID,
+            TOPICS_V0);
+
+    // V1 removes max_num_offsets
+    private static final Field PARTITIONS_V1 = PARTITIONS.withFields(
+            PARTITION_ID,
+            TIMESTAMP);
+
+    private static final Field TOPICS_V1 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V1);
+
     private static final Schema LIST_OFFSET_REQUEST_V1 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(LIST_OFFSET_REQUEST_TOPIC_V1), "Topics to list offsets."));
+            REPLICA_ID,
+            TOPICS_V1);
 
+    // V2 adds a field for the isolation level
     private static final Schema LIST_OFFSET_REQUEST_V2 = new Schema(
-            new Field(REPLICA_ID_KEY_NAME, INT32, "Broker id of the follower. For normal consumers, use -1."),
-            new Field(ISOLATION_LEVEL_KEY_NAME, INT8, "This setting controls the visibility of transactional records. " +
-                    "Using READ_UNCOMMITTED (isolation_level = 0) makes all records visible. With READ_COMMITTED " +
-                    "(isolation_level = 1), non-transactional and COMMITTED transactional records are visible. " +
-                    "To be more concrete, READ_COMMITTED returns all data from offsets smaller than the current " +
-                    "LSO (last stable offset), and enables the inclusion of the list of aborted transactions in the " +
-                    "result, which allows consumers to discard ABORTED transactional records"),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(LIST_OFFSET_REQUEST_TOPIC_V1), "Topics to list offsets."));;
+            REPLICA_ID,
+            ISOLATION_LEVEL,
+            TOPICS_V1);
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V3 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema LIST_OFFSET_REQUEST_V3 = LIST_OFFSET_REQUEST_V2;
 
+    // V4 introduces the current leader epoch, which is used for fencing
+    private static final Field PARTITIONS_V4 = PARTITIONS.withFields(
+            PARTITION_ID,
+            CURRENT_LEADER_EPOCH,
+            TIMESTAMP);
+
+    private static final Field TOPICS_V4 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V4);
+
+    private static final Schema LIST_OFFSET_REQUEST_V4 = new Schema(
+            REPLICA_ID,
+            ISOLATION_LEVEL,
+            TOPICS_V4);
+
     public static Schema[] schemaVersions() {
         return new Schema[] {LIST_OFFSET_REQUEST_V0, LIST_OFFSET_REQUEST_V1, LIST_OFFSET_REQUEST_V2,
-            LIST_OFFSET_REQUEST_V3};
+            LIST_OFFSET_REQUEST_V3, LIST_OFFSET_REQUEST_V4};
     }
 
     private final int replicaId;
     private final IsolationLevel isolationLevel;
-    private final Map offsetData;
-    private final Map partitionTimestamps;
+    private final Map partitionTimestamps;
     private final Set duplicatePartitions;
 
     public static class Builder extends AbstractRequest.Builder {
         private final int replicaId;
         private final IsolationLevel isolationLevel;
-        private Map offsetData = null;
-        private Map partitionTimestamps = null;
+        private Map partitionTimestamps = new HashMap<>();
 
         public static Builder forReplica(short allowedVersion, int replicaId) {
             return new Builder((short) 0, allowedVersion, replicaId, IsolationLevel.READ_UNCOMMITTED);
@@ -125,49 +146,23 @@ else if (requireTimestamp)
             return new Builder(minVersion, ApiKeys.LIST_OFFSETS.latestVersion(), CONSUMER_REPLICA_ID, isolationLevel);
         }
 
-        private Builder(short oldestAllowedVersion, short latestAllowedVersion, int replicaId, IsolationLevel isolationLevel) {
+        private Builder(short oldestAllowedVersion,
+                        short latestAllowedVersion,
+                        int replicaId,
+                        IsolationLevel isolationLevel) {
             super(ApiKeys.LIST_OFFSETS, oldestAllowedVersion, latestAllowedVersion);
             this.replicaId = replicaId;
             this.isolationLevel = isolationLevel;
         }
 
-        public Builder setOffsetData(Map offsetData) {
-            this.offsetData = offsetData;
-            return this;
-        }
-
-        public Builder setTargetTimes(Map partitionTimestamps) {
+        public Builder setTargetTimes(Map partitionTimestamps) {
             this.partitionTimestamps = partitionTimestamps;
             return this;
         }
 
         @Override
         public ListOffsetRequest build(short version) {
-            if (version == 0) {
-                if (offsetData == null) {
-                    if (partitionTimestamps == null) {
-                        throw new RuntimeException("Must set partitionTimestamps or offsetData when creating a v0 " +
-                            "ListOffsetRequest");
-                    } else {
-                        offsetData = new HashMap<>();
-                        for (Map.Entry entry: partitionTimestamps.entrySet()) {
-                            offsetData.put(entry.getKey(),
-                                    new PartitionData(entry.getValue(), 1));
-                        }
-                        this.partitionTimestamps = null;
-                    }
-                }
-            } else {
-                if (offsetData != null) {
-                    throw new RuntimeException("Cannot create a v" + version + " ListOffsetRequest with v0 " +
-                        "PartitionData.");
-                } else if (partitionTimestamps == null) {
-                    throw new RuntimeException("Must set partitionTimestamps when creating a v" +
-                            version + " ListOffsetRequest");
-                }
-            }
-            Map m = (version == 0) ?  offsetData : partitionTimestamps;
-            return new ListOffsetRequest(replicaId, m, isolationLevel, version);
+            return new ListOffsetRequest(replicaId, partitionTimestamps, isolationLevel, version);
         }
 
         @Override
@@ -175,9 +170,6 @@ public String toString() {
             StringBuilder bld = new StringBuilder();
             bld.append("(type=ListOffsetRequest")
                .append(", replicaId=").append(replicaId);
-            if (offsetData != null) {
-                bld.append(", offsetData=").append(offsetData);
-            }
             if (partitionTimestamps != null) {
                 bld.append(", partitionTimestamps=").append(partitionTimestamps);
             }
@@ -187,25 +179,34 @@ public String toString() {
         }
     }
 
-    /**
-     * This class is only used by ListOffsetRequest v0 which has been deprecated.
-     */
-    @Deprecated
     public static final class PartitionData {
         public final long timestamp;
-        public final int maxNumOffsets;
+        @Deprecated
+        public final int maxNumOffsets; // only supported in v0
+        public final Optional currentLeaderEpoch;
 
-        public PartitionData(long timestamp, int maxNumOffsets) {
+        private PartitionData(long timestamp, int maxNumOffsets, Optional currentLeaderEpoch) {
             this.timestamp = timestamp;
             this.maxNumOffsets = maxNumOffsets;
+            this.currentLeaderEpoch = currentLeaderEpoch;
+        }
+
+        @Deprecated
+        public PartitionData(long timestamp, int maxNumOffsets) {
+            this(timestamp, maxNumOffsets, Optional.empty());
+        }
+
+        public PartitionData(long timestamp, Optional currentLeaderEpoch) {
+            this(timestamp, 1, currentLeaderEpoch);
         }
 
         @Override
         public String toString() {
             StringBuilder bld = new StringBuilder();
             bld.append("{timestamp: ").append(timestamp).
-                append(", maxNumOffsets: ").append(maxNumOffsets).
-                append("}");
+                    append(", maxNumOffsets: ").append(maxNumOffsets).
+                    append(", currentLeaderEpoch: ").append(currentLeaderEpoch).
+                    append("}");
             return bld.toString();
         }
     }
@@ -214,40 +215,39 @@ public String toString() {
      * Private constructor with a specified version.
      */
     @SuppressWarnings("unchecked")
-    private ListOffsetRequest(int replicaId, Map targetTimes, IsolationLevel isolationLevel, short version) {
-        super(version);
+    private ListOffsetRequest(int replicaId,
+                              Map targetTimes,
+                              IsolationLevel isolationLevel,
+                              short version) {
+        super(ApiKeys.LIST_OFFSETS, version);
         this.replicaId = replicaId;
         this.isolationLevel = isolationLevel;
-        this.offsetData = version == 0 ? (Map) targetTimes : null;
-        this.partitionTimestamps = version >= 1 ? (Map) targetTimes : null;
+        this.partitionTimestamps = targetTimes;
         this.duplicatePartitions = Collections.emptySet();
     }
 
     public ListOffsetRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.LIST_OFFSETS, version);
         Set duplicatePartitions = new HashSet<>();
-        replicaId = struct.getInt(REPLICA_ID_KEY_NAME);
-        isolationLevel = struct.hasField(ISOLATION_LEVEL_KEY_NAME) ?
-                IsolationLevel.forId(struct.getByte(ISOLATION_LEVEL_KEY_NAME)) :
+        replicaId = struct.get(REPLICA_ID);
+        isolationLevel = struct.hasField(ISOLATION_LEVEL) ?
+                IsolationLevel.forId(struct.get(ISOLATION_LEVEL)) :
                 IsolationLevel.READ_UNCOMMITTED;
-        offsetData = new HashMap<>();
         partitionTimestamps = new HashMap<>();
-        for (Object topicResponseObj : struct.getArray(TOPICS_KEY_NAME)) {
+        for (Object topicResponseObj : struct.get(TOPICS)) {
             Struct topicResponse = (Struct) topicResponseObj;
             String topic = topicResponse.get(TOPIC_NAME);
-            for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
                 Struct partitionResponse = (Struct) partitionResponseObj;
                 int partition = partitionResponse.get(PARTITION_ID);
-                long timestamp = partitionResponse.getLong(TIMESTAMP_KEY_NAME);
+                long timestamp = partitionResponse.get(TIMESTAMP);
                 TopicPartition tp = new TopicPartition(topic, partition);
-                if (partitionResponse.hasField(MAX_NUM_OFFSETS_KEY_NAME)) {
-                    int maxNumOffsets = partitionResponse.getInt(MAX_NUM_OFFSETS_KEY_NAME);
-                    PartitionData partitionData = new PartitionData(timestamp, maxNumOffsets);
-                    offsetData.put(tp, partitionData);
-                } else {
-                    if (partitionTimestamps.put(tp, timestamp) != null)
-                        duplicatePartitions.add(tp);
-                }
+
+                int maxNumOffsets = partitionResponse.getOrElse(MAX_NUM_OFFSETS, 1);
+                Optional currentLeaderEpoch = RequestUtils.getLeaderEpoch(partitionResponse, CURRENT_LEADER_EPOCH);
+                PartitionData partitionData = new PartitionData(timestamp, maxNumOffsets, currentLeaderEpoch);
+                if (partitionTimestamps.put(tp, partitionData) != null)
+                    duplicatePartitions.add(tp);
             }
         }
         this.duplicatePartitions = duplicatePartitions;
@@ -257,27 +257,21 @@ public ListOffsetRequest(Struct struct, short version) {
     @SuppressWarnings("deprecation")
     public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         Map responseData = new HashMap<>();
-
         short versionId = version();
-        if (versionId == 0) {
-            for (Map.Entry entry : offsetData.entrySet()) {
-                ListOffsetResponse.PartitionData partitionResponse = new ListOffsetResponse.PartitionData(
-                        Errors.forException(e), Collections.emptyList());
-                responseData.put(entry.getKey(), partitionResponse);
-            }
-        } else {
-            for (Map.Entry entry : partitionTimestamps.entrySet()) {
-                ListOffsetResponse.PartitionData partitionResponse = new ListOffsetResponse.PartitionData(
-                        Errors.forException(e), -1L, -1L);
-                responseData.put(entry.getKey(), partitionResponse);
-            }
+
+        ListOffsetResponse.PartitionData partitionError = versionId == 0 ?
+                new ListOffsetResponse.PartitionData(Errors.forException(e), Collections.emptyList()) :
+                new ListOffsetResponse.PartitionData(Errors.forException(e), -1L, -1L, Optional.empty());
+        for (TopicPartition partition : partitionTimestamps.keySet()) {
+            responseData.put(partition, partitionError);
         }
 
-        switch (versionId) {
+        switch (version()) {
             case 0:
             case 1:
             case 2:
             case 3:
+            case 4:
                 return new ListOffsetResponse(throttleTimeMs, responseData);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
@@ -293,12 +287,7 @@ public IsolationLevel isolationLevel() {
         return isolationLevel;
     }
 
-    @Deprecated
-    public Map offsetData() {
-        return offsetData;
-    }
-
-    public Map partitionTimestamps() {
+    public Map partitionTimestamps() {
         return partitionTimestamps;
     }
 
@@ -314,39 +303,30 @@ public static ListOffsetRequest parse(ByteBuffer buffer, short version) {
     protected Struct toStruct() {
         short version = version();
         Struct struct = new Struct(ApiKeys.LIST_OFFSETS.requestSchema(version));
+        Map> topicsData = CollectionUtils.groupPartitionDataByTopic(partitionTimestamps);
 
-        Map targetTimes = partitionTimestamps == null ? offsetData : partitionTimestamps;
-        Map> topicsData = CollectionUtils.groupDataByTopic(targetTimes);
-
-        struct.set(REPLICA_ID_KEY_NAME, replicaId);
+        struct.set(REPLICA_ID, replicaId);
+        struct.setIfExists(ISOLATION_LEVEL, isolationLevel.id());
 
-        if (struct.hasField(ISOLATION_LEVEL_KEY_NAME))
-            struct.set(ISOLATION_LEVEL_KEY_NAME, isolationLevel.id());
         List topicArray = new ArrayList<>();
-        for (Map.Entry> topicEntry: topicsData.entrySet()) {
-            Struct topicData = struct.instance(TOPICS_KEY_NAME);
+        for (Map.Entry> topicEntry: topicsData.entrySet()) {
+            Struct topicData = struct.instance(TOPICS);
             topicData.set(TOPIC_NAME, topicEntry.getKey());
             List partitionArray = new ArrayList<>();
-            for (Map.Entry partitionEntry : topicEntry.getValue().entrySet()) {
-                if (version == 0) {
-                    PartitionData offsetPartitionData = (PartitionData) partitionEntry.getValue();
-                    Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
-                    partitionData.set(PARTITION_ID, partitionEntry.getKey());
-                    partitionData.set(TIMESTAMP_KEY_NAME, offsetPartitionData.timestamp);
-                    partitionData.set(MAX_NUM_OFFSETS_KEY_NAME, offsetPartitionData.maxNumOffsets);
-                    partitionArray.add(partitionData);
-                } else {
-                    Long timestamp = (Long) partitionEntry.getValue();
-                    Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
-                    partitionData.set(PARTITION_ID, partitionEntry.getKey());
-                    partitionData.set(TIMESTAMP_KEY_NAME, timestamp);
-                    partitionArray.add(partitionData);
-                }
+            for (Map.Entry partitionEntry : topicEntry.getValue().entrySet()) {
+                PartitionData offsetPartitionData = partitionEntry.getValue();
+                Struct partitionData = topicData.instance(PARTITIONS);
+                partitionData.set(PARTITION_ID, partitionEntry.getKey());
+                partitionData.set(TIMESTAMP, offsetPartitionData.timestamp);
+                partitionData.setIfExists(MAX_NUM_OFFSETS, offsetPartitionData.maxNumOffsets);
+                RequestUtils.setLeaderEpochIfExists(partitionData, CURRENT_LEADER_EPOCH,
+                        offsetPartitionData.currentLeaderEpoch);
+                partitionArray.add(partitionData);
             }
-            topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+            topicData.set(PARTITIONS, partitionArray.toArray());
             topicArray.add(topicData);
         }
-        struct.set(TOPICS_KEY_NAME, topicArray.toArray());
+        struct.set(TOPICS, topicArray.toArray());
         return struct;
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
index e719dbbdbfd11..9f3ce73a004ab 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -31,73 +30,98 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
+import static org.apache.kafka.common.protocol.CommonFields.LEADER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
 import static org.apache.kafka.common.protocol.types.Type.INT64;
 
+/**
+ * Possible error code:
+ *
+ *  UNKNOWN_TOPIC_OR_PARTITION (3)
+ *  NOT_LEADER_FOR_PARTITION (6)
+ *  UNSUPPORTED_FOR_MESSAGE_FORMAT (43)
+ *  UNKNOWN (-1)
+ */
 public class ListOffsetResponse extends AbstractResponse {
     public static final long UNKNOWN_TIMESTAMP = -1L;
     public static final long UNKNOWN_OFFSET = -1L;
 
-    private static final String RESPONSES_KEY_NAME = "responses";
+    // top level fields
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("responses",
+            "The listed offsets by topic");
 
-    // topic level field names
-    private static final String PARTITIONS_KEY_NAME = "partition_responses";
-
-    /**
-     * Possible error code:
-     *
-     *  UNKNOWN_TOPIC_OR_PARTITION (3)
-     *  NOT_LEADER_FOR_PARTITION (6)
-     *  UNSUPPORTED_FOR_MESSAGE_FORMAT (43)
-     *  UNKNOWN (-1)
-     */
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partition_responses",
+            "The listed offsets by partition");
 
+    // partition level fields
     // This key is only used by ListOffsetResponse v0
     @Deprecated
-    private static final String OFFSETS_KEY_NAME = "offsets";
-    private static final String TIMESTAMP_KEY_NAME = "timestamp";
-    private static final String OFFSET_KEY_NAME = "offset";
+    private static final Field.Array OFFSETS = new Field.Array("offsets'", INT64, "A list of offsets.");
+    private static final Field.Int64 TIMESTAMP = new Field.Int64("timestamp",
+            "The timestamp associated with the returned offset");
+    private static final Field.Int64 OFFSET = new Field.Int64("offset",
+            "The offset found");
 
-    private static final Schema LIST_OFFSET_RESPONSE_PARTITION_V0 = new Schema(
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
             ERROR_CODE,
-            new Field(OFFSETS_KEY_NAME, new ArrayOf(INT64), "A list of offsets."));
+            OFFSETS);
+
+    private static final Field TOPICS_V0 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V0);
 
-    private static final Schema LIST_OFFSET_RESPONSE_PARTITION_V1 = new Schema(
+    private static final Schema LIST_OFFSET_RESPONSE_V0 = new Schema(
+            TOPICS_V0);
+
+    // V1 bumped for the removal of the offsets array
+    private static final Field PARTITIONS_V1 = PARTITIONS.withFields(
             PARTITION_ID,
             ERROR_CODE,
-            new Field(TIMESTAMP_KEY_NAME, INT64, "The timestamp associated with the returned offset"),
-            new Field(OFFSET_KEY_NAME, INT64, "offset found"));
-
-    private static final Schema LIST_OFFSET_RESPONSE_TOPIC_V0 = new Schema(
-            TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(LIST_OFFSET_RESPONSE_PARTITION_V0)));
+            TIMESTAMP,
+            OFFSET);
 
-    private static final Schema LIST_OFFSET_RESPONSE_TOPIC_V1 = new Schema(
+    private static final Field TOPICS_V1 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(LIST_OFFSET_RESPONSE_PARTITION_V1)));
-
-    private static final Schema LIST_OFFSET_RESPONSE_V0 = new Schema(
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(LIST_OFFSET_RESPONSE_TOPIC_V0)));
+            PARTITIONS_V1);
 
     private static final Schema LIST_OFFSET_RESPONSE_V1 = new Schema(
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(LIST_OFFSET_RESPONSE_TOPIC_V1)));
+            TOPICS_V1);
+
+    // V2 bumped for the addition of the throttle time
     private static final Schema LIST_OFFSET_RESPONSE_V2 = new Schema(
             THROTTLE_TIME_MS,
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(LIST_OFFSET_RESPONSE_TOPIC_V1)));
+            TOPICS_V1);
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V3 bumped to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema LIST_OFFSET_RESPONSE_V3 = LIST_OFFSET_RESPONSE_V2;
 
+    // V4 bumped for the addition of the current leader epoch in the request schema and the
+    // leader epoch in the response partition data
+    private static final Field PARTITIONS_V4 = PARTITIONS.withFields(
+            PARTITION_ID,
+            ERROR_CODE,
+            TIMESTAMP,
+            OFFSET,
+            LEADER_EPOCH);
+
+    private static final Field TOPICS_V4 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V4);
+
+    private static final Schema LIST_OFFSET_RESPONSE_V4 = new Schema(
+            THROTTLE_TIME_MS,
+            TOPICS_V4);
+
     public static Schema[] schemaVersions() {
         return new Schema[] {LIST_OFFSET_RESPONSE_V0, LIST_OFFSET_RESPONSE_V1, LIST_OFFSET_RESPONSE_V2,
-            LIST_OFFSET_RESPONSE_V3};
+            LIST_OFFSET_RESPONSE_V3, LIST_OFFSET_RESPONSE_V4};
     }
 
     public static final class PartitionData {
@@ -107,6 +131,7 @@ public static final class PartitionData {
         public final List offsets;
         public final Long timestamp;
         public final Long offset;
+        public final Optional leaderEpoch;
 
         /**
          * Constructor for ListOffsetResponse v0
@@ -117,32 +142,37 @@ public PartitionData(Errors error, List offsets) {
             this.offsets = offsets;
             this.timestamp = null;
             this.offset = null;
+            this.leaderEpoch = Optional.empty();
         }
 
         /**
          * Constructor for ListOffsetResponse v1
          */
-        public PartitionData(Errors error, long timestamp, long offset) {
+        public PartitionData(Errors error, long timestamp, long offset, Optional leaderEpoch) {
             this.error = error;
             this.timestamp = timestamp;
             this.offset = offset;
             this.offsets = null;
+            this.leaderEpoch = leaderEpoch;
         }
 
         @Override
         public String toString() {
             StringBuilder bld = new StringBuilder();
-            bld.append("PartitionData{").
-                append("errorCode: ").append((int) error.code()).
-                append(", timestamp: ").append(timestamp).
-                append(", offset: ").append(offset).
-                append(", offsets: ");
+            bld.append("PartitionData(").
+                    append("errorCode: ").append((int) error.code());
+
             if (offsets == null) {
-                bld.append(offsets);
+                bld.append(", timestamp: ").append(timestamp).
+                        append(", offset: ").append(offset).
+                        append(", leaderEpoch: ").append(leaderEpoch);
             } else {
-                bld.append("[").append(Utils.join(this.offsets, ",")).append("]");
+                bld.append(", offsets: ").
+                        append("[").
+                        append(Utils.join(this.offsets, ",")).
+                        append("]");
             }
-            bld.append("}");
+            bld.append(")");
             return bld.toString();
         }
     }
@@ -165,24 +195,25 @@ public ListOffsetResponse(int throttleTimeMs, Map
     public ListOffsetResponse(Struct struct) {
         this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
         responseData = new HashMap<>();
-        for (Object topicResponseObj : struct.getArray(RESPONSES_KEY_NAME)) {
+        for (Object topicResponseObj : struct.get(TOPICS)) {
             Struct topicResponse = (Struct) topicResponseObj;
             String topic = topicResponse.get(TOPIC_NAME);
-            for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
                 Struct partitionResponse = (Struct) partitionResponseObj;
                 int partition = partitionResponse.get(PARTITION_ID);
                 Errors error = Errors.forCode(partitionResponse.get(ERROR_CODE));
                 PartitionData partitionData;
-                if (partitionResponse.hasField(OFFSETS_KEY_NAME)) {
-                    Object[] offsets = partitionResponse.getArray(OFFSETS_KEY_NAME);
+                if (partitionResponse.hasField(OFFSETS)) {
+                    Object[] offsets = partitionResponse.get(OFFSETS);
                     List offsetsList = new ArrayList<>();
                     for (Object offset : offsets)
                         offsetsList.add((Long) offset);
                     partitionData = new PartitionData(error, offsetsList);
                 } else {
-                    long timestamp = partitionResponse.getLong(TIMESTAMP_KEY_NAME);
-                    long offset = partitionResponse.getLong(OFFSET_KEY_NAME);
-                    partitionData = new PartitionData(error, timestamp, offset);
+                    long timestamp = partitionResponse.get(TIMESTAMP);
+                    long offset = partitionResponse.get(OFFSET);
+                    Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionResponse, LEADER_EPOCH);
+                    partitionData = new PartitionData(error, timestamp, offset, leaderEpoch);
                 }
                 responseData.put(new TopicPartition(topic, partition), partitionData);
             }
@@ -214,30 +245,31 @@ public static ListOffsetResponse parse(ByteBuffer buffer, short version) {
     protected Struct toStruct(short version) {
         Struct struct = new Struct(ApiKeys.LIST_OFFSETS.responseSchema(version));
         struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
-        Map> topicsData = CollectionUtils.groupDataByTopic(responseData);
+        Map> topicsData = CollectionUtils.groupPartitionDataByTopic(responseData);
 
         List topicArray = new ArrayList<>();
         for (Map.Entry> topicEntry: topicsData.entrySet()) {
-            Struct topicData = struct.instance(RESPONSES_KEY_NAME);
+            Struct topicData = struct.instance(TOPICS);
             topicData.set(TOPIC_NAME, topicEntry.getKey());
             List partitionArray = new ArrayList<>();
             for (Map.Entry partitionEntry : topicEntry.getValue().entrySet()) {
                 PartitionData offsetPartitionData = partitionEntry.getValue();
-                Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+                Struct partitionData = topicData.instance(PARTITIONS);
                 partitionData.set(PARTITION_ID, partitionEntry.getKey());
                 partitionData.set(ERROR_CODE, offsetPartitionData.error.code());
-                if (version == 0)
-                    partitionData.set(OFFSETS_KEY_NAME, offsetPartitionData.offsets.toArray());
-                else {
-                    partitionData.set(TIMESTAMP_KEY_NAME, offsetPartitionData.timestamp);
-                    partitionData.set(OFFSET_KEY_NAME, offsetPartitionData.offset);
+                if (version == 0) {
+                    partitionData.set(OFFSETS, offsetPartitionData.offsets.toArray());
+                } else {
+                    partitionData.set(TIMESTAMP, offsetPartitionData.timestamp);
+                    partitionData.set(OFFSET, offsetPartitionData.offset);
+                    RequestUtils.setLeaderEpochIfExists(partitionData, LEADER_EPOCH, offsetPartitionData.leaderEpoch);
                 }
                 partitionArray.add(partitionData);
             }
-            topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+            topicData.set(PARTITIONS, partitionArray.toArray());
             topicArray.add(topicData);
         }
-        struct.set(RESPONSES_KEY_NAME, topicArray.toArray());
+        struct.set(TOPICS, topicArray.toArray());
 
         return struct;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
index 67dbe94af40a7..89a6e69359cb8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.Node;
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
@@ -31,13 +30,11 @@
 import java.util.Collections;
 import java.util.List;
 
-import static org.apache.kafka.common.protocol.types.Type.BOOLEAN;
 import static org.apache.kafka.common.protocol.types.Type.STRING;
 
 public class MetadataRequest extends AbstractRequest {
 
     private static final String TOPICS_KEY_NAME = "topics";
-    private static final String ALLOW_AUTO_TOPIC_CREATION_KEY_NAME = "allow_auto_topic_creation";
 
     private static final Schema METADATA_REQUEST_V0 = new Schema(
             new Field(TOPICS_KEY_NAME, new ArrayOf(STRING), "An array of topics to fetch metadata for. If no topics are specified fetch metadata for all topics."));
@@ -52,12 +49,14 @@ public class MetadataRequest extends AbstractRequest {
     private static final Schema METADATA_REQUEST_V3 = METADATA_REQUEST_V2;
 
     /* The v4 metadata request has an additional field for allowing auto topic creation. The response is the same as v3. */
+    private static final Field.Bool ALLOW_AUTO_TOPIC_CREATION = new Field.Bool("allow_auto_topic_creation",
+            "If this and the broker config auto.create.topics.enable are true, topics that " +
+                    "don't exist will be created by the broker. Otherwise, no topics will be created by the broker.");
+
     private static final Schema METADATA_REQUEST_V4 = new Schema(
             new Field(TOPICS_KEY_NAME, ArrayOf.nullable(STRING), "An array of topics to fetch metadata for. " +
                     "If the topics array is null fetch metadata for all topics."),
-            new Field(ALLOW_AUTO_TOPIC_CREATION_KEY_NAME, BOOLEAN, "If this and the broker config " +
-                    "'auto.create.topics.enable' are true, topics that don't exist will be created by the broker. " +
-                    "Otherwise, no topics will be created by the broker."));
+            ALLOW_AUTO_TOPIC_CREATION);
 
     /* The v5 metadata request is the same as v4. An additional field for offline_replicas has been added to the v5 metadata response */
     private static final Schema METADATA_REQUEST_V5 = METADATA_REQUEST_V4;
@@ -67,9 +66,14 @@ public class MetadataRequest extends AbstractRequest {
      */
     private static final Schema METADATA_REQUEST_V6 = METADATA_REQUEST_V5;
 
+    /**
+     * Bumped for the addition of the current leader epoch in the metadata response.
+     */
+    private static final Schema METADATA_REQUEST_V7 = METADATA_REQUEST_V6;
+
     public static Schema[] schemaVersions() {
         return new Schema[] {METADATA_REQUEST_V0, METADATA_REQUEST_V1, METADATA_REQUEST_V2, METADATA_REQUEST_V3,
-            METADATA_REQUEST_V4, METADATA_REQUEST_V5, METADATA_REQUEST_V6};
+            METADATA_REQUEST_V4, METADATA_REQUEST_V5, METADATA_REQUEST_V6, METADATA_REQUEST_V7};
     }
 
     public static class Builder extends AbstractRequest.Builder {
@@ -133,13 +137,13 @@ public String toString() {
      * In v1 null indicates requesting all topics, and an empty list indicates requesting no topics.
      */
     public MetadataRequest(List topics, boolean allowAutoTopicCreation, short version) {
-        super(version);
+        super(ApiKeys.METADATA, version);
         this.topics = topics;
         this.allowAutoTopicCreation = allowAutoTopicCreation;
     }
 
     public MetadataRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.METADATA, version);
         Object[] topicArray = struct.getArray(TOPICS_KEY_NAME);
         if (topicArray != null) {
             topics = new ArrayList<>();
@@ -149,10 +153,8 @@ public MetadataRequest(Struct struct, short version) {
         } else {
             topics = null;
         }
-        if (struct.hasField(ALLOW_AUTO_TOPIC_CREATION_KEY_NAME))
-            allowAutoTopicCreation = struct.getBoolean(ALLOW_AUTO_TOPIC_CREATION_KEY_NAME);
-        else
-            allowAutoTopicCreation = true;
+
+        allowAutoTopicCreation = struct.getOrElse(ALLOW_AUTO_TOPIC_CREATION, true);
     }
 
     @Override
@@ -171,12 +173,13 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 0:
             case 1:
             case 2:
-                return new MetadataResponse(Collections.emptyList(), null, MetadataResponse.NO_CONTROLLER_ID, topicMetadatas);
+                return new MetadataResponse(Collections.emptyList(), null, MetadataResponse.NO_CONTROLLER_ID, topicMetadatas);
             case 3:
             case 4:
             case 5:
             case 6:
-                return new MetadataResponse(throttleTimeMs, Collections.emptyList(), null, MetadataResponse.NO_CONTROLLER_ID, topicMetadatas);
+            case 7:
+                return new MetadataResponse(throttleTimeMs, Collections.emptyList(), null, MetadataResponse.NO_CONTROLLER_ID, topicMetadatas);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
                         versionId, this.getClass().getSimpleName(), ApiKeys.METADATA.latestVersion()));
@@ -206,8 +209,7 @@ protected Struct toStruct() {
             struct.set(TOPICS_KEY_NAME, null);
         else
             struct.set(TOPICS_KEY_NAME, topics.toArray());
-        if (struct.hasField(ALLOW_AUTO_TOPIC_CREATION_KEY_NAME))
-            struct.set(ALLOW_AUTO_TOPIC_CREATION_KEY_NAME, allowAutoTopicCreation);
+        struct.setIfExists(ALLOW_AUTO_TOPIC_CREATION, allowAutoTopicCreation);
         return struct;
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
index 09a04e5560321..c78066f93bba2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
@@ -22,7 +22,6 @@
 import org.apache.kafka.common.errors.InvalidMetadataException;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -35,143 +34,170 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
+import static org.apache.kafka.common.protocol.CommonFields.LEADER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.BOOLEAN;
 import static org.apache.kafka.common.protocol.types.Type.INT32;
-import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
-import static org.apache.kafka.common.protocol.types.Type.STRING;
 
-public class MetadataResponse extends AbstractResponse {
-    private static final String BROKERS_KEY_NAME = "brokers";
-    private static final String TOPIC_METADATA_KEY_NAME = "topic_metadata";
-
-    // broker level field names
-    private static final String NODE_ID_KEY_NAME = "node_id";
-    private static final String HOST_KEY_NAME = "host";
-    private static final String PORT_KEY_NAME = "port";
-    private static final String RACK_KEY_NAME = "rack";
+/**
+ * Possible topic-level error codes:
+ *  UnknownTopic (3)
+ *  LeaderNotAvailable (5)
+ *  InvalidTopic (17)
+ *  TopicAuthorizationFailed (29)
 
-    private static final String CONTROLLER_ID_KEY_NAME = "controller_id";
+ * Possible partition-level error codes:
+ *  LeaderNotAvailable (5)
+ *  ReplicaNotAvailable (9)
+ */
+public class MetadataResponse extends AbstractResponse {
     public static final int NO_CONTROLLER_ID = -1;
 
-    private static final String CLUSTER_ID_KEY_NAME = "cluster_id";
-
-    /**
-     * Possible error codes:
-     *
-     * UnknownTopic (3)
-     * LeaderNotAvailable (5)
-     * InvalidTopic (17)
-     * TopicAuthorizationFailed (29)
-     */
-
-    private static final String IS_INTERNAL_KEY_NAME = "is_internal";
-    private static final String PARTITION_METADATA_KEY_NAME = "partition_metadata";
-
-    /**
-     * Possible error codes:
-     *
-     * LeaderNotAvailable (5)
-     * ReplicaNotAvailable (9)
-     */
-
-    private static final String LEADER_KEY_NAME = "leader";
-    private static final String REPLICAS_KEY_NAME = "replicas";
-    private static final String ISR_KEY_NAME = "isr";
-    private static final String OFFLINE_REPLICAS_KEY_NAME = "offline_replicas";
-
-    private static final Schema METADATA_BROKER_V0 = new Schema(
-            new Field(NODE_ID_KEY_NAME, INT32, "The broker id."),
-            new Field(HOST_KEY_NAME, STRING, "The hostname of the broker."),
-            new Field(PORT_KEY_NAME, INT32, "The port on which the broker accepts requests."));
-
-    private static final Schema PARTITION_METADATA_V0 = new Schema(
+    private static final Field.ComplexArray BROKERS = new Field.ComplexArray("brokers",
+            "Host and port information for all brokers.");
+    private static final Field.ComplexArray TOPIC_METADATA = new Field.ComplexArray("topic_metadata",
+            "Metadata for requested topics");
+
+    // cluster level fields
+    private static final Field.NullableStr CLUSTER_ID = new Field.NullableStr("cluster_id",
+            "The cluster id that this broker belongs to.");
+    private static final Field.Int32 CONTROLLER_ID = new Field.Int32("controller_id",
+            "The broker id of the controller broker.");
+
+    // broker level fields
+    private static final Field.Int32 NODE_ID = new Field.Int32("node_id", "The broker id.");
+    private static final Field.Str HOST = new Field.Str("host", "The hostname of the broker.");
+    private static final Field.Int32 PORT = new Field.Int32("port", "The port on which the broker accepts requests.");
+    private static final Field.NullableStr RACK = new Field.NullableStr("rack", "The rack of the broker.");
+
+    // topic level fields
+    private static final Field.ComplexArray PARTITION_METADATA = new Field.ComplexArray("partition_metadata",
+            "Metadata for each partition of the topic.");
+    private static final Field.Bool IS_INTERNAL = new Field.Bool("is_internal",
+            "Indicates if the topic is considered a Kafka internal topic");
+
+    // partition level fields
+    private static final Field.Int32 LEADER = new Field.Int32("leader",
+            "The id of the broker acting as leader for this partition.");
+    private static final Field.Array REPLICAS = new Field.Array("replicas", INT32,
+            "The set of all nodes that host this partition.");
+    private static final Field.Array ISR = new Field.Array("isr", INT32,
+            "The set of nodes that are in sync with the leader for this partition.");
+    private static final Field.Array OFFLINE_REPLICAS = new Field.Array("offline_replicas", INT32,
+            "The set of offline replicas of this partition.");
+
+    private static final Field METADATA_BROKER_V0 = BROKERS.withFields(
+            NODE_ID,
+            HOST,
+            PORT);
+
+    private static final Field PARTITION_METADATA_V0 = PARTITION_METADATA.withFields(
             ERROR_CODE,
             PARTITION_ID,
-            new Field(LEADER_KEY_NAME, INT32, "The id of the broker acting as leader for this partition."),
-            new Field(REPLICAS_KEY_NAME, new ArrayOf(INT32), "The set of all nodes that host this partition."),
-            new Field(ISR_KEY_NAME, new ArrayOf(INT32), "The set of nodes that are in sync with the leader for this partition."));
+            LEADER,
+            REPLICAS,
+            ISR);
 
-    private static final Schema TOPIC_METADATA_V0 = new Schema(
+    private static final Field TOPIC_METADATA_V0 = TOPIC_METADATA.withFields(
             ERROR_CODE,
             TOPIC_NAME,
-            new Field(PARTITION_METADATA_KEY_NAME, new ArrayOf(PARTITION_METADATA_V0), "Metadata for each partition of the topic."));
+            PARTITION_METADATA_V0);
 
     private static final Schema METADATA_RESPONSE_V0 = new Schema(
-            new Field(BROKERS_KEY_NAME, new ArrayOf(METADATA_BROKER_V0), "Host and port information for all brokers."),
-            new Field(TOPIC_METADATA_KEY_NAME, new ArrayOf(TOPIC_METADATA_V0)));
+            METADATA_BROKER_V0,
+            TOPIC_METADATA_V0);
 
-    private static final Schema METADATA_BROKER_V1 = new Schema(
-            new Field(NODE_ID_KEY_NAME, INT32, "The broker id."),
-            new Field(HOST_KEY_NAME, STRING, "The hostname of the broker."),
-            new Field(PORT_KEY_NAME, INT32, "The port on which the broker accepts requests."),
-            new Field(RACK_KEY_NAME, NULLABLE_STRING, "The rack of the broker."));
+    // V1 adds fields for the rack of each broker, the controller id, and whether or not the topic is internal
+    private static final Field METADATA_BROKER_V1 = BROKERS.withFields(
+            NODE_ID,
+            HOST,
+            PORT,
+            RACK);
 
-    private static final Schema PARTITION_METADATA_V1 = PARTITION_METADATA_V0;
-
-    // PARTITION_METADATA_V2 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline.
-    private static final Schema PARTITION_METADATA_V2 = new Schema(
-            ERROR_CODE,
-            PARTITION_ID,
-            new Field(LEADER_KEY_NAME, INT32, "The id of the broker acting as leader for this partition."),
-            new Field(REPLICAS_KEY_NAME, new ArrayOf(INT32), "The set of all nodes that host this partition."),
-            new Field(ISR_KEY_NAME, new ArrayOf(INT32), "The set of nodes that are in sync with the leader for this partition."),
-            new Field(OFFLINE_REPLICAS_KEY_NAME, new ArrayOf(INT32), "The set of offline replicas of this partition."));
-
-    private static final Schema TOPIC_METADATA_V1 = new Schema(
+    private static final Field TOPIC_METADATA_V1 = TOPIC_METADATA.withFields(
             ERROR_CODE,
             TOPIC_NAME,
-            new Field(IS_INTERNAL_KEY_NAME, BOOLEAN, "Indicates if the topic is considered a Kafka internal topic"),
-            new Field(PARTITION_METADATA_KEY_NAME, new ArrayOf(PARTITION_METADATA_V1), "Metadata for each partition of the topic."));
-
-    // TOPIC_METADATA_V2 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline.
-    private static final Schema TOPIC_METADATA_V2 = new Schema(
-            ERROR_CODE,
-            TOPIC_NAME,
-            new Field(IS_INTERNAL_KEY_NAME, BOOLEAN, "Indicates if the topic is considered a Kafka internal topic"),
-            new Field(PARTITION_METADATA_KEY_NAME, new ArrayOf(PARTITION_METADATA_V2), "Metadata for each partition of the topic."));
+            IS_INTERNAL,
+            PARTITION_METADATA_V0);
 
     private static final Schema METADATA_RESPONSE_V1 = new Schema(
-            new Field(BROKERS_KEY_NAME, new ArrayOf(METADATA_BROKER_V1), "Host and port information for all brokers."),
-            new Field(CONTROLLER_ID_KEY_NAME, INT32, "The broker id of the controller broker."),
-            new Field(TOPIC_METADATA_KEY_NAME, new ArrayOf(TOPIC_METADATA_V1)));
+            METADATA_BROKER_V1,
+            CONTROLLER_ID,
+            TOPIC_METADATA_V1);
 
+    // V2 added a field for the cluster id
     private static final Schema METADATA_RESPONSE_V2 = new Schema(
-            new Field(BROKERS_KEY_NAME, new ArrayOf(METADATA_BROKER_V1), "Host and port information for all brokers."),
-            new Field(CLUSTER_ID_KEY_NAME, NULLABLE_STRING, "The cluster id that this broker belongs to."),
-            new Field(CONTROLLER_ID_KEY_NAME, INT32, "The broker id of the controller broker."),
-            new Field(TOPIC_METADATA_KEY_NAME, new ArrayOf(TOPIC_METADATA_V1)));
+            METADATA_BROKER_V1,
+            CLUSTER_ID,
+            CONTROLLER_ID,
+            TOPIC_METADATA_V1);
 
+    // V3 adds the throttle time to the response
     private static final Schema METADATA_RESPONSE_V3 = new Schema(
             THROTTLE_TIME_MS,
-            new Field(BROKERS_KEY_NAME, new ArrayOf(METADATA_BROKER_V1), "Host and port information for all brokers."),
-            new Field(CLUSTER_ID_KEY_NAME, NULLABLE_STRING, "The cluster id that this broker belongs to."),
-            new Field(CONTROLLER_ID_KEY_NAME, INT32, "The broker id of the controller broker."),
-            new Field(TOPIC_METADATA_KEY_NAME, new ArrayOf(TOPIC_METADATA_V1)));
+            METADATA_BROKER_V1,
+            CLUSTER_ID,
+            CONTROLLER_ID,
+            TOPIC_METADATA_V1);
 
     private static final Schema METADATA_RESPONSE_V4 = METADATA_RESPONSE_V3;
 
-    // METADATA_RESPONSE_V5 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline.
+    // V5 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline.
+    private static final Field PARTITION_METADATA_V5 = PARTITION_METADATA.withFields(
+            ERROR_CODE,
+            PARTITION_ID,
+            LEADER,
+            REPLICAS,
+            ISR,
+            OFFLINE_REPLICAS);
+
+    private static final Field TOPIC_METADATA_V5 = TOPIC_METADATA.withFields(
+            ERROR_CODE,
+            TOPIC_NAME,
+            IS_INTERNAL,
+            PARTITION_METADATA_V5);
+
     private static final Schema METADATA_RESPONSE_V5 = new Schema(
             THROTTLE_TIME_MS,
-            new Field(BROKERS_KEY_NAME, new ArrayOf(METADATA_BROKER_V1), "Host and port information for all brokers."),
-            new Field(CLUSTER_ID_KEY_NAME, NULLABLE_STRING, "The cluster id that this broker belongs to."),
-            new Field(CONTROLLER_ID_KEY_NAME, INT32, "The broker id of the controller broker."),
-            new Field(TOPIC_METADATA_KEY_NAME, new ArrayOf(TOPIC_METADATA_V2)));
+            METADATA_BROKER_V1,
+            CLUSTER_ID,
+            CONTROLLER_ID,
+            TOPIC_METADATA_V5);
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V6 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema METADATA_RESPONSE_V6 = METADATA_RESPONSE_V5;
 
+    // V7 adds the leader epoch to the partition metadata
+    private static final Field PARTITION_METADATA_V7 = PARTITION_METADATA.withFields(
+            ERROR_CODE,
+            PARTITION_ID,
+            LEADER,
+            LEADER_EPOCH,
+            REPLICAS,
+            ISR,
+            OFFLINE_REPLICAS);
+
+    private static final Field TOPIC_METADATA_V7 = TOPIC_METADATA.withFields(
+            ERROR_CODE,
+            TOPIC_NAME,
+            IS_INTERNAL,
+            PARTITION_METADATA_V7);
+
+    private static final Schema METADATA_RESPONSE_V7 = new Schema(
+            THROTTLE_TIME_MS,
+            METADATA_BROKER_V1,
+            CLUSTER_ID,
+            CONTROLLER_ID,
+            TOPIC_METADATA_V7);
+
     public static Schema[] schemaVersions() {
         return new Schema[] {METADATA_RESPONSE_V0, METADATA_RESPONSE_V1, METADATA_RESPONSE_V2, METADATA_RESPONSE_V3,
-            METADATA_RESPONSE_V4, METADATA_RESPONSE_V5, METADATA_RESPONSE_V6};
+            METADATA_RESPONSE_V4, METADATA_RESPONSE_V5, METADATA_RESPONSE_V6, METADATA_RESPONSE_V7};
     }
 
     private final int throttleTimeMs;
@@ -198,62 +224,56 @@ public MetadataResponse(int throttleTimeMs, List brokers, String clusterId
     public MetadataResponse(Struct struct) {
         this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
         Map brokers = new HashMap<>();
-        Object[] brokerStructs = (Object[]) struct.get(BROKERS_KEY_NAME);
+        Object[] brokerStructs = struct.get(BROKERS);
         for (Object brokerStruct : brokerStructs) {
             Struct broker = (Struct) brokerStruct;
-            int nodeId = broker.getInt(NODE_ID_KEY_NAME);
-            String host = broker.getString(HOST_KEY_NAME);
-            int port = broker.getInt(PORT_KEY_NAME);
+            int nodeId = broker.get(NODE_ID);
+            String host = broker.get(HOST);
+            int port = broker.get(PORT);
             // This field only exists in v1+
             // When we can't know if a rack exists in a v0 response we default to null
-            String rack = broker.hasField(RACK_KEY_NAME) ? broker.getString(RACK_KEY_NAME) : null;
+            String rack = broker.getOrElse(RACK, null);
             brokers.put(nodeId, new Node(nodeId, host, port, rack));
         }
 
         // This field only exists in v1+
         // When we can't know the controller id in a v0 response we default to NO_CONTROLLER_ID
-        int controllerId = NO_CONTROLLER_ID;
-        if (struct.hasField(CONTROLLER_ID_KEY_NAME))
-            controllerId = struct.getInt(CONTROLLER_ID_KEY_NAME);
+        int controllerId = struct.getOrElse(CONTROLLER_ID, NO_CONTROLLER_ID);
 
         // This field only exists in v2+
-        if (struct.hasField(CLUSTER_ID_KEY_NAME)) {
-            this.clusterId = struct.getString(CLUSTER_ID_KEY_NAME);
-        } else {
-            this.clusterId = null;
-        }
+        this.clusterId = struct.getOrElse(CLUSTER_ID, null);
 
         List topicMetadata = new ArrayList<>();
-        Object[] topicInfos = (Object[]) struct.get(TOPIC_METADATA_KEY_NAME);
+        Object[] topicInfos = struct.get(TOPIC_METADATA);
         for (Object topicInfoObj : topicInfos) {
             Struct topicInfo = (Struct) topicInfoObj;
             Errors topicError = Errors.forCode(topicInfo.get(ERROR_CODE));
             String topic = topicInfo.get(TOPIC_NAME);
             // This field only exists in v1+
             // When we can't know if a topic is internal or not in a v0 response we default to false
-            boolean isInternal = topicInfo.hasField(IS_INTERNAL_KEY_NAME) ? topicInfo.getBoolean(IS_INTERNAL_KEY_NAME) : false;
-
+            boolean isInternal = topicInfo.getOrElse(IS_INTERNAL, false);
             List partitionMetadata = new ArrayList<>();
 
-            Object[] partitionInfos = (Object[]) topicInfo.get(PARTITION_METADATA_KEY_NAME);
+            Object[] partitionInfos = topicInfo.get(PARTITION_METADATA);
             for (Object partitionInfoObj : partitionInfos) {
                 Struct partitionInfo = (Struct) partitionInfoObj;
                 Errors partitionError = Errors.forCode(partitionInfo.get(ERROR_CODE));
                 int partition = partitionInfo.get(PARTITION_ID);
-                int leader = partitionInfo.getInt(LEADER_KEY_NAME);
+                int leader = partitionInfo.get(LEADER);
+                Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionInfo, LEADER_EPOCH);
                 Node leaderNode = leader == -1 ? null : brokers.get(leader);
 
-                Object[] replicas = (Object[]) partitionInfo.get(REPLICAS_KEY_NAME);
+                Object[] replicas = partitionInfo.get(REPLICAS);
                 List replicaNodes = convertToNodes(brokers, replicas);
 
-                Object[] isr = (Object[]) partitionInfo.get(ISR_KEY_NAME);
+                Object[] isr = partitionInfo.get(ISR);
                 List isrNodes = convertToNodes(brokers, isr);
 
-                Object[] offlineReplicas = partitionInfo.hasField(OFFLINE_REPLICAS_KEY_NAME) ?
-                    (Object[]) partitionInfo.get(OFFLINE_REPLICAS_KEY_NAME) : new Object[0];
+                Object[] offlineReplicas = partitionInfo.getOrEmpty(OFFLINE_REPLICAS);
                 List offlineNodes = convertToNodes(brokers, offlineReplicas);
 
-                partitionMetadata.add(new PartitionMetadata(partitionError, partition, leaderNode, replicaNodes, isrNodes, offlineNodes));
+                partitionMetadata.add(new PartitionMetadata(partitionError, partition, leaderNode, leaderEpoch,
+                        replicaNodes, isrNodes, offlineNodes));
             }
 
             topicMetadata.add(new TopicMetadata(topicError, topic, isInternal, partitionMetadata));
@@ -354,7 +374,7 @@ public Cluster cluster() {
             if (metadata.error == Errors.NONE) {
                 if (metadata.isInternal)
                     internalTopics.add(metadata.topic);
-                for (PartitionMetadata partitionMetadata : metadata.partitionMetadata)
+                for (PartitionMetadata partitionMetadata : metadata.partitionMetadata) {
                     partitions.add(new PartitionInfo(
                             metadata.topic,
                             partitionMetadata.partition,
@@ -362,6 +382,7 @@ public Cluster cluster() {
                             partitionMetadata.replicas.toArray(new Node[0]),
                             partitionMetadata.isr.toArray(new Node[0]),
                             partitionMetadata.offlineReplicas.toArray(new Node[0])));
+                }
             }
         }
 
@@ -452,6 +473,7 @@ public static class PartitionMetadata {
         private final Errors error;
         private final int partition;
         private final Node leader;
+        private final Optional leaderEpoch;
         private final List replicas;
         private final List isr;
         private final List offlineReplicas;
@@ -459,12 +481,14 @@ public static class PartitionMetadata {
         public PartitionMetadata(Errors error,
                                  int partition,
                                  Node leader,
+                                 Optional leaderEpoch,
                                  List replicas,
                                  List isr,
                                  List offlineReplicas) {
             this.error = error;
             this.partition = partition;
             this.leader = leader;
+            this.leaderEpoch = leaderEpoch;
             this.replicas = replicas;
             this.isr = isr;
             this.offlineReplicas = offlineReplicas;
@@ -482,6 +506,10 @@ public int leaderId() {
             return leader == null ? -1 : leader.id();
         }
 
+        public Optional leaderEpoch() {
+            return leaderEpoch;
+        }
+
         public Node leader() {
             return leader;
         }
@@ -504,6 +532,7 @@ public String toString() {
                     ", error=" + error +
                     ", partition=" + partition +
                     ", leader=" + leader +
+                    ", leaderEpoch=" + leaderEpoch +
                     ", replicas=" + Utils.join(replicas, ",") +
                     ", isr=" + Utils.join(isr, ",") +
                     ", offlineReplicas=" + Utils.join(offlineReplicas, ",") + ')';
@@ -516,61 +545,61 @@ protected Struct toStruct(short version) {
         struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
         List brokerArray = new ArrayList<>();
         for (Node node : brokers) {
-            Struct broker = struct.instance(BROKERS_KEY_NAME);
-            broker.set(NODE_ID_KEY_NAME, node.id());
-            broker.set(HOST_KEY_NAME, node.host());
-            broker.set(PORT_KEY_NAME, node.port());
+            Struct broker = struct.instance(BROKERS);
+            broker.set(NODE_ID, node.id());
+            broker.set(HOST, node.host());
+            broker.set(PORT, node.port());
             // This field only exists in v1+
-            if (broker.hasField(RACK_KEY_NAME))
-                broker.set(RACK_KEY_NAME, node.rack());
+            broker.setIfExists(RACK, node.rack());
             brokerArray.add(broker);
         }
-        struct.set(BROKERS_KEY_NAME, brokerArray.toArray());
+        struct.set(BROKERS, brokerArray.toArray());
 
         // This field only exists in v1+
-        if (struct.hasField(CONTROLLER_ID_KEY_NAME))
-            struct.set(CONTROLLER_ID_KEY_NAME, controller == null ? NO_CONTROLLER_ID : controller.id());
+        struct.setIfExists(CONTROLLER_ID, controller == null ? NO_CONTROLLER_ID : controller.id());
 
         // This field only exists in v2+
-        if (struct.hasField(CLUSTER_ID_KEY_NAME))
-            struct.set(CLUSTER_ID_KEY_NAME, clusterId);
+        struct.setIfExists(CLUSTER_ID, clusterId);
 
         List topicMetadataArray = new ArrayList<>(topicMetadata.size());
         for (TopicMetadata metadata : topicMetadata) {
-            Struct topicData = struct.instance(TOPIC_METADATA_KEY_NAME);
+            Struct topicData = struct.instance(TOPIC_METADATA);
             topicData.set(TOPIC_NAME, metadata.topic);
             topicData.set(ERROR_CODE, metadata.error.code());
             // This field only exists in v1+
-            if (topicData.hasField(IS_INTERNAL_KEY_NAME))
-                topicData.set(IS_INTERNAL_KEY_NAME, metadata.isInternal());
+            topicData.setIfExists(IS_INTERNAL, metadata.isInternal());
 
             List partitionMetadataArray = new ArrayList<>(metadata.partitionMetadata.size());
             for (PartitionMetadata partitionMetadata : metadata.partitionMetadata()) {
-                Struct partitionData = topicData.instance(PARTITION_METADATA_KEY_NAME);
+                Struct partitionData = topicData.instance(PARTITION_METADATA);
                 partitionData.set(ERROR_CODE, partitionMetadata.error.code());
                 partitionData.set(PARTITION_ID, partitionMetadata.partition);
-                partitionData.set(LEADER_KEY_NAME, partitionMetadata.leaderId());
+                partitionData.set(LEADER, partitionMetadata.leaderId());
+
+                // Leader epoch exists in v7 forward
+                RequestUtils.setLeaderEpochIfExists(partitionData, LEADER_EPOCH, partitionMetadata.leaderEpoch);
+
                 ArrayList replicas = new ArrayList<>(partitionMetadata.replicas.size());
                 for (Node node : partitionMetadata.replicas)
                     replicas.add(node.id());
-                partitionData.set(REPLICAS_KEY_NAME, replicas.toArray());
+                partitionData.set(REPLICAS, replicas.toArray());
                 ArrayList isr = new ArrayList<>(partitionMetadata.isr.size());
                 for (Node node : partitionMetadata.isr)
                     isr.add(node.id());
-                partitionData.set(ISR_KEY_NAME, isr.toArray());
-                if (partitionData.hasField(OFFLINE_REPLICAS_KEY_NAME)) {
+                partitionData.set(ISR, isr.toArray());
+                if (partitionData.hasField(OFFLINE_REPLICAS)) {
                     ArrayList offlineReplicas = new ArrayList<>(partitionMetadata.offlineReplicas.size());
                     for (Node node : partitionMetadata.offlineReplicas)
                         offlineReplicas.add(node.id());
-                    partitionData.set(OFFLINE_REPLICAS_KEY_NAME, offlineReplicas.toArray());
+                    partitionData.set(OFFLINE_REPLICAS, offlineReplicas.toArray());
                 }
                 partitionMetadataArray.add(partitionData);
 
             }
-            topicData.set(PARTITION_METADATA_KEY_NAME, partitionMetadataArray.toArray());
+            topicData.set(PARTITION_METADATA, partitionMetadataArray.toArray());
             topicMetadataArray.add(topicData);
         }
-        struct.set(TOPIC_METADATA_KEY_NAME, topicMetadataArray.toArray());
+        struct.set(TOPIC_METADATA, topicMetadataArray.toArray());
         return struct;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
index 8a51e84e76a17..fdb5b1081996d 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
@@ -17,10 +17,8 @@
 package org.apache.kafka.common.requests;
 
 import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -31,95 +29,112 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_LEADER_EPOCH;
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_METADATA;
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_OFFSET;
 import static org.apache.kafka.common.protocol.CommonFields.GENERATION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.GROUP_ID;
 import static org.apache.kafka.common.protocol.CommonFields.MEMBER_ID;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
 
-/**
- * This wrapper supports both v0 and v1 of OffsetCommitRequest.
- */
 public class OffsetCommitRequest extends AbstractRequest {
-    private static final String TOPICS_KEY_NAME = "topics";
-    private static final String RETENTION_TIME_KEY_NAME = "retention_time";
-
-    // topic level field names
-    private static final String PARTITIONS_KEY_NAME = "partitions";
+    // top level fields
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "Topics to commit offsets");
 
-    // partition level field names
-    private static final String COMMIT_OFFSET_KEY_NAME = "offset";
-    private static final String METADATA_KEY_NAME = "metadata";
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "Partitions to commit offsets");
 
+    // partition level fields
     @Deprecated
-    private static final String TIMESTAMP_KEY_NAME = "timestamp";         // for v0, v1
+    private static final Field.Int64 COMMIT_TIMESTAMP = new Field.Int64("timestamp", "Timestamp of the commit");
+    private static final Field.Int64 RETENTION_TIME = new Field.Int64("retention_time",
+            "Time period in ms to retain the offset.");
 
-    /* Offset commit api */
-    private static final Schema OFFSET_COMMIT_REQUEST_PARTITION_V0 = new Schema(
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
-            new Field(COMMIT_OFFSET_KEY_NAME, INT64, "Message offset to be committed."),
-            new Field(METADATA_KEY_NAME, NULLABLE_STRING, "Any associated metadata the client wants to keep."));
+            COMMITTED_OFFSET,
+            COMMITTED_METADATA);
 
-    private static final Schema OFFSET_COMMIT_REQUEST_PARTITION_V1 = new Schema(
-            PARTITION_ID,
-            new Field(COMMIT_OFFSET_KEY_NAME, INT64, "Message offset to be committed."),
-            new Field(TIMESTAMP_KEY_NAME, INT64, "Timestamp of the commit"),
-            new Field(METADATA_KEY_NAME, NULLABLE_STRING, "Any associated metadata the client wants to keep."));
-
-    private static final Schema OFFSET_COMMIT_REQUEST_PARTITION_V2 = new Schema(
-            PARTITION_ID,
-            new Field(COMMIT_OFFSET_KEY_NAME, INT64, "Message offset to be committed."),
-            new Field(METADATA_KEY_NAME, NULLABLE_STRING, "Any associated metadata the client wants to keep."));
-
-    private static final Schema OFFSET_COMMIT_REQUEST_TOPIC_V0 = new Schema(
-            TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_PARTITION_V0), "Partitions to commit offsets."));
-
-    private static final Schema OFFSET_COMMIT_REQUEST_TOPIC_V1 = new Schema(
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_PARTITION_V1), "Partitions to commit offsets."));
-
-    private static final Schema OFFSET_COMMIT_REQUEST_TOPIC_V2 = new Schema(
-            TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_PARTITION_V2), "Partitions to commit offsets."));
+            PARTITIONS_V0);
 
     private static final Schema OFFSET_COMMIT_REQUEST_V0 = new Schema(
             GROUP_ID,
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_TOPIC_V0), "Topics to commit offsets."));
+            TOPICS_V0);
+
+    // V1 adds timestamp and group membership information (generation and memberId)
+    private static final Field PARTITIONS_V1 = PARTITIONS.withFields(
+            PARTITION_ID,
+            COMMITTED_OFFSET,
+            COMMIT_TIMESTAMP,
+            COMMITTED_METADATA);
+
+    private static final Field TOPICS_V1 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V1);
 
     private static final Schema OFFSET_COMMIT_REQUEST_V1 = new Schema(
             GROUP_ID,
             GENERATION_ID,
             MEMBER_ID,
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_TOPIC_V1), "Topics to commit offsets."));
+            TOPICS_V1);
+
+    // V2 adds retention time
+    private static final Field PARTITIONS_V2 = PARTITIONS.withFields(
+            PARTITION_ID,
+            COMMITTED_OFFSET,
+            COMMITTED_METADATA);
+
+    private static final Field TOPICS_V2 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V2);
 
     private static final Schema OFFSET_COMMIT_REQUEST_V2 = new Schema(
             GROUP_ID,
             GENERATION_ID,
             MEMBER_ID,
-            new Field(RETENTION_TIME_KEY_NAME, INT64, "Time period in ms to retain the offset."),
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_TOPIC_V2), "Topics to commit offsets."));
+            RETENTION_TIME,
+            TOPICS_V2);
 
-    /* v3 request is same as v2. Throttle time has been added to response */
+    // V3 adds throttle time
     private static final Schema OFFSET_COMMIT_REQUEST_V3 = OFFSET_COMMIT_REQUEST_V2;
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V4 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema OFFSET_COMMIT_REQUEST_V4 = OFFSET_COMMIT_REQUEST_V3;
 
+    // V5 removes the retention time which is now controlled only by a broker configuration
     private static final Schema OFFSET_COMMIT_REQUEST_V5 = new Schema(
             GROUP_ID,
             GENERATION_ID,
             MEMBER_ID,
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_REQUEST_TOPIC_V2), "Topics to commit offsets."));
+            TOPICS_V2);
+
+    // V6 adds the leader epoch to the partition data
+    private static final Field PARTITIONS_V6 = PARTITIONS.withFields(
+            PARTITION_ID,
+            COMMITTED_OFFSET,
+            COMMITTED_LEADER_EPOCH,
+            COMMITTED_METADATA);
+
+    private static final Field TOPICS_V6 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V6);
+
+    private static final Schema OFFSET_COMMIT_REQUEST_V6 = new Schema(
+            GROUP_ID,
+            GENERATION_ID,
+            MEMBER_ID,
+            TOPICS_V6);
 
     public static Schema[] schemaVersions() {
         return new Schema[] {OFFSET_COMMIT_REQUEST_V0, OFFSET_COMMIT_REQUEST_V1, OFFSET_COMMIT_REQUEST_V2,
-            OFFSET_COMMIT_REQUEST_V3, OFFSET_COMMIT_REQUEST_V4, OFFSET_COMMIT_REQUEST_V5};
+            OFFSET_COMMIT_REQUEST_V3, OFFSET_COMMIT_REQUEST_V4, OFFSET_COMMIT_REQUEST_V5, OFFSET_COMMIT_REQUEST_V6};
     }
 
     // default values for the current version
@@ -144,25 +159,32 @@ public static final class PartitionData {
 
         public final long offset;
         public final String metadata;
+        public final Optional leaderEpoch;
 
-        @Deprecated
-        public PartitionData(long offset, long timestamp, String metadata) {
+        private PartitionData(long offset, Optional leaderEpoch, long timestamp, String metadata) {
             this.offset = offset;
+            this.leaderEpoch = leaderEpoch;
             this.timestamp = timestamp;
             this.metadata = metadata;
         }
 
-        public PartitionData(long offset, String metadata) {
-            this(offset, DEFAULT_TIMESTAMP, metadata);
+        @Deprecated
+        public PartitionData(long offset, long timestamp, String metadata) {
+            this(offset, Optional.empty(), timestamp, metadata);
+        }
+
+        public PartitionData(long offset, Optional leaderEpoch, String metadata) {
+            this(offset, leaderEpoch, DEFAULT_TIMESTAMP, metadata);
         }
 
         @Override
         public String toString() {
             StringBuilder bld = new StringBuilder();
             bld.append("(timestamp=").append(timestamp).
-                append(", offset=").append(offset).
-                append(", metadata=").append(metadata).
-                append(")");
+                    append(", offset=").append(offset).
+                    append(", leaderEpoch=").append(leaderEpoch).
+                    append(", metadata=").append(metadata).
+                    append(")");
             return bld.toString();
         }
     }
@@ -191,18 +213,12 @@ public Builder setGenerationId(int generationId) {
 
         @Override
         public OffsetCommitRequest build(short version) {
-            switch (version) {
-                case 0:
-                    return new OffsetCommitRequest(groupId, DEFAULT_GENERATION_ID, DEFAULT_MEMBER_ID,
-                            DEFAULT_RETENTION_TIME, offsetData, version);
-                case 1:
-                case 2:
-                case 3:
-                case 4:
-                case 5:
-                    return new OffsetCommitRequest(groupId, generationId, memberId, DEFAULT_RETENTION_TIME, offsetData, version);
-                default:
-                    throw new UnsupportedVersionException("Unsupported version " + version);
+            if (version == 0) {
+                return new OffsetCommitRequest(groupId, DEFAULT_GENERATION_ID, DEFAULT_MEMBER_ID,
+                        DEFAULT_RETENTION_TIME, offsetData, version);
+            } else {
+                return new OffsetCommitRequest(groupId, generationId, memberId, DEFAULT_RETENTION_TIME,
+                        offsetData, version);
             }
         }
 
@@ -221,7 +237,7 @@ public String toString() {
 
     private OffsetCommitRequest(String groupId, int generationId, String memberId, long retentionTime,
                                 Map offsetData, short version) {
-        super(version);
+        super(ApiKeys.OFFSET_COMMIT, version);
         this.groupId = groupId;
         this.generationId = generationId;
         this.memberId = memberId;
@@ -230,7 +246,7 @@ private OffsetCommitRequest(String groupId, int generationId, String memberId, l
     }
 
     public OffsetCommitRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.OFFSET_COMMIT, versionId);
 
         groupId = struct.get(GROUP_ID);
 
@@ -239,27 +255,26 @@ public OffsetCommitRequest(Struct struct, short versionId) {
         memberId = struct.getOrElse(MEMBER_ID, DEFAULT_MEMBER_ID);
 
         // This field only exists in v2
-        if (struct.hasField(RETENTION_TIME_KEY_NAME))
-            retentionTime = struct.getLong(RETENTION_TIME_KEY_NAME);
-        else
-            retentionTime = DEFAULT_RETENTION_TIME;
+        retentionTime = struct.getOrElse(RETENTION_TIME, DEFAULT_RETENTION_TIME);
 
         offsetData = new HashMap<>();
-        for (Object topicDataObj : struct.getArray(TOPICS_KEY_NAME)) {
+        for (Object topicDataObj : struct.get(TOPICS)) {
             Struct topicData = (Struct) topicDataObj;
             String topic = topicData.get(TOPIC_NAME);
-            for (Object partitionDataObj : topicData.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionDataObj : topicData.get(PARTITIONS)) {
                 Struct partitionDataStruct = (Struct) partitionDataObj;
                 int partition = partitionDataStruct.get(PARTITION_ID);
-                long offset = partitionDataStruct.getLong(COMMIT_OFFSET_KEY_NAME);
-                String metadata = partitionDataStruct.getString(METADATA_KEY_NAME);
+                long offset = partitionDataStruct.get(COMMITTED_OFFSET);
+                String metadata = partitionDataStruct.get(COMMITTED_METADATA);
                 PartitionData partitionOffset;
                 // This field only exists in v1
-                if (partitionDataStruct.hasField(TIMESTAMP_KEY_NAME)) {
-                    long timestamp = partitionDataStruct.getLong(TIMESTAMP_KEY_NAME);
+                if (partitionDataStruct.hasField(COMMIT_TIMESTAMP)) {
+                    long timestamp = partitionDataStruct.get(COMMIT_TIMESTAMP);
                     partitionOffset = new PartitionData(offset, timestamp, metadata);
                 } else {
-                    partitionOffset = new PartitionData(offset, metadata);
+                    Optional leaderEpochOpt = RequestUtils.getLeaderEpoch(partitionDataStruct,
+                            COMMITTED_LEADER_EPOCH);
+                    partitionOffset = new PartitionData(offset, leaderEpochOpt, metadata);
                 }
                 offsetData.put(new TopicPartition(topic, partition), partitionOffset);
             }
@@ -272,31 +287,31 @@ public Struct toStruct() {
         Struct struct = new Struct(ApiKeys.OFFSET_COMMIT.requestSchema(version));
         struct.set(GROUP_ID, groupId);
 
-        Map> topicsData = CollectionUtils.groupDataByTopic(offsetData);
+        Map> topicsData = CollectionUtils.groupPartitionDataByTopic(offsetData);
         List topicArray = new ArrayList<>();
         for (Map.Entry> topicEntry: topicsData.entrySet()) {
-            Struct topicData = struct.instance(TOPICS_KEY_NAME);
+            Struct topicData = struct.instance(TOPICS);
             topicData.set(TOPIC_NAME, topicEntry.getKey());
             List partitionArray = new ArrayList<>();
             for (Map.Entry partitionEntry : topicEntry.getValue().entrySet()) {
                 PartitionData fetchPartitionData = partitionEntry.getValue();
-                Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+                Struct partitionData = topicData.instance(PARTITIONS);
                 partitionData.set(PARTITION_ID, partitionEntry.getKey());
-                partitionData.set(COMMIT_OFFSET_KEY_NAME, fetchPartitionData.offset);
+                partitionData.set(COMMITTED_OFFSET, fetchPartitionData.offset);
                 // Only for v1
-                if (partitionData.hasField(TIMESTAMP_KEY_NAME))
-                    partitionData.set(TIMESTAMP_KEY_NAME, fetchPartitionData.timestamp);
-                partitionData.set(METADATA_KEY_NAME, fetchPartitionData.metadata);
+                partitionData.setIfExists(COMMIT_TIMESTAMP, fetchPartitionData.timestamp);
+                // Only for v6
+                RequestUtils.setLeaderEpochIfExists(partitionData, COMMITTED_LEADER_EPOCH, fetchPartitionData.leaderEpoch);
+                partitionData.set(COMMITTED_METADATA, fetchPartitionData.metadata);
                 partitionArray.add(partitionData);
             }
-            topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+            topicData.set(PARTITIONS, partitionArray.toArray());
             topicArray.add(topicData);
         }
-        struct.set(TOPICS_KEY_NAME, topicArray.toArray());
+        struct.set(TOPICS, topicArray.toArray());
         struct.setIfExists(GENERATION_ID, generationId);
         struct.setIfExists(MEMBER_ID, memberId);
-        if (struct.hasField(RETENTION_TIME_KEY_NAME))
-            struct.set(RETENTION_TIME_KEY_NAME, retentionTime);
+        struct.setIfExists(RETENTION_TIME, retentionTime);
         return struct;
     }
 
@@ -316,6 +331,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 3:
             case 4:
             case 5:
+            case 6:
                 return new OffsetCommitResponse(throttleTimeMs, responseData);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
@@ -335,6 +351,7 @@ public String memberId() {
         return memberId;
     }
 
+    @Deprecated
     public long retentionTime() {
         return retentionTime;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index c79bc57885b0b..4d724a32813bc 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -36,60 +35,64 @@
 import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
 
+/**
+ * Possible error codes:
+ *
+ * UNKNOWN_TOPIC_OR_PARTITION (3)
+ * REQUEST_TIMED_OUT (7)
+ * OFFSET_METADATA_TOO_LARGE (12)
+ * COORDINATOR_LOAD_IN_PROGRESS (14)
+ * GROUP_COORDINATOR_NOT_AVAILABLE (15)
+ * NOT_COORDINATOR (16)
+ * ILLEGAL_GENERATION (22)
+ * UNKNOWN_MEMBER_ID (25)
+ * REBALANCE_IN_PROGRESS (27)
+ * INVALID_COMMIT_OFFSET_SIZE (28)
+ * TOPIC_AUTHORIZATION_FAILED (29)
+ * GROUP_AUTHORIZATION_FAILED (30)
+ */
 public class OffsetCommitResponse extends AbstractResponse {
-
-    private static final String RESPONSES_KEY_NAME = "responses";
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("responses",
+            "Responses by topic for committed partitions");
 
     // topic level fields
-    private static final String PARTITIONS_KEY_NAME = "partition_responses";
-
-    /**
-     * Possible error codes:
-     *
-     * UNKNOWN_TOPIC_OR_PARTITION (3)
-     * REQUEST_TIMED_OUT (7)
-     * OFFSET_METADATA_TOO_LARGE (12)
-     * COORDINATOR_LOAD_IN_PROGRESS (14)
-     * GROUP_COORDINATOR_NOT_AVAILABLE (15)
-     * NOT_COORDINATOR (16)
-     * ILLEGAL_GENERATION (22)
-     * UNKNOWN_MEMBER_ID (25)
-     * REBALANCE_IN_PROGRESS (27)
-     * INVALID_COMMIT_OFFSET_SIZE (28)
-     * TOPIC_AUTHORIZATION_FAILED (29)
-     * GROUP_AUTHORIZATION_FAILED (30)
-     */
-
-    private static final Schema OFFSET_COMMIT_RESPONSE_PARTITION_V0 = new Schema(
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partition_responses",
+            "Responses for committed partitions");
+
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
             ERROR_CODE);
 
-    private static final Schema OFFSET_COMMIT_RESPONSE_TOPIC_V0 = new Schema(
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_COMMIT_RESPONSE_PARTITION_V0)));
+            PARTITIONS_V0);
 
     private static final Schema OFFSET_COMMIT_RESPONSE_V0 = new Schema(
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(OFFSET_COMMIT_RESPONSE_TOPIC_V0)));
+            TOPICS_V0);
 
-
-    /* The response types for V0, V1 and V2 of OFFSET_COMMIT_REQUEST are the same. */
+    // V1 adds timestamp and group membership information (generation and memberId) to the request
     private static final Schema OFFSET_COMMIT_RESPONSE_V1 = OFFSET_COMMIT_RESPONSE_V0;
+
+    // V2 adds retention time to the request
     private static final Schema OFFSET_COMMIT_RESPONSE_V2 = OFFSET_COMMIT_RESPONSE_V0;
 
+    // V3 adds throttle time
     private static final Schema OFFSET_COMMIT_RESPONSE_V3 = new Schema(
             THROTTLE_TIME_MS,
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(OFFSET_COMMIT_RESPONSE_TOPIC_V0)));
+            TOPICS_V0);
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V4 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema OFFSET_COMMIT_RESPONSE_V4 = OFFSET_COMMIT_RESPONSE_V3;
 
+    // V5 removes retention time from the request
     private static final Schema OFFSET_COMMIT_RESPONSE_V5 = OFFSET_COMMIT_RESPONSE_V4;
 
+    // V6 adds leader epoch to the request
+    private static final Schema OFFSET_COMMIT_RESPONSE_V6 = OFFSET_COMMIT_RESPONSE_V5;
+
     public static Schema[] schemaVersions() {
         return new Schema[] {OFFSET_COMMIT_RESPONSE_V0, OFFSET_COMMIT_RESPONSE_V1, OFFSET_COMMIT_RESPONSE_V2,
-            OFFSET_COMMIT_RESPONSE_V3, OFFSET_COMMIT_RESPONSE_V4, OFFSET_COMMIT_RESPONSE_V5};
+            OFFSET_COMMIT_RESPONSE_V3, OFFSET_COMMIT_RESPONSE_V4, OFFSET_COMMIT_RESPONSE_V5, OFFSET_COMMIT_RESPONSE_V6};
     }
 
     private final Map responseData;
@@ -107,10 +110,10 @@ public OffsetCommitResponse(int throttleTimeMs, Map resp
     public OffsetCommitResponse(Struct struct) {
         this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
         responseData = new HashMap<>();
-        for (Object topicResponseObj : struct.getArray(RESPONSES_KEY_NAME)) {
+        for (Object topicResponseObj : struct.get(TOPICS)) {
             Struct topicResponse = (Struct) topicResponseObj;
             String topic = topicResponse.get(TOPIC_NAME);
-            for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
                 Struct partitionResponse = (Struct) partitionResponseObj;
                 int partition = partitionResponse.get(PARTITION_ID);
                 Errors error = Errors.forCode(partitionResponse.get(ERROR_CODE));
@@ -124,22 +127,22 @@ public Struct toStruct(short version) {
         Struct struct = new Struct(ApiKeys.OFFSET_COMMIT.responseSchema(version));
         struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
 
-        Map> topicsData = CollectionUtils.groupDataByTopic(responseData);
+        Map> topicsData = CollectionUtils.groupPartitionDataByTopic(responseData);
         List topicArray = new ArrayList<>();
         for (Map.Entry> entries: topicsData.entrySet()) {
-            Struct topicData = struct.instance(RESPONSES_KEY_NAME);
+            Struct topicData = struct.instance(TOPICS);
             topicData.set(TOPIC_NAME, entries.getKey());
             List partitionArray = new ArrayList<>();
             for (Map.Entry partitionEntry : entries.getValue().entrySet()) {
-                Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+                Struct partitionData = topicData.instance(PARTITIONS);
                 partitionData.set(PARTITION_ID, partitionEntry.getKey());
                 partitionData.set(ERROR_CODE, partitionEntry.getValue().code());
                 partitionArray.add(partitionData);
             }
-            topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+            topicData.set(PARTITIONS, partitionArray.toArray());
             topicArray.add(topicData);
         }
-        struct.set(RESPONSES_KEY_NAME, topicArray.toArray());
+        struct.set(TOPICS, topicArray.toArray());
 
         return struct;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
index e90c6ebbe916e..d2f5c888cec19 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.common.errors.UnsupportedVersionException;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -32,16 +31,20 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.kafka.common.protocol.CommonFields.GROUP_ID;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
 
 public class OffsetFetchRequest extends AbstractRequest {
-    private static final String TOPICS_KEY_NAME = "topics";
+    // top level fields
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "Topics to fetch offsets. If the topic array is null fetch offsets for all topics.");
 
-    // topic level field names
-    private static final String PARTITIONS_KEY_NAME = "partitions";
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "Partitions to fetch offsets.");
 
     /*
      * Wire formats of version 0 and 1 are the same, but with different functionality.
@@ -54,32 +57,41 @@ public class OffsetFetchRequest extends AbstractRequest {
      * a 'null' is passed instead of a list of specific topic partitions. It also returns a top level error code
      * for group or coordinator level errors.
      */
-    private static final Schema OFFSET_FETCH_REQUEST_PARTITION_V0 = new Schema(
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID);
 
-    private static final Schema OFFSET_FETCH_REQUEST_TOPIC_V0 = new Schema(
+    private static final Field TOPICS_V0 = TOPICS.withFields("Topics to fetch offsets.",
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_FETCH_REQUEST_PARTITION_V0), "Partitions to fetch offsets."));
+            PARTITIONS_V0);
 
     private static final Schema OFFSET_FETCH_REQUEST_V0 = new Schema(
             GROUP_ID,
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_FETCH_REQUEST_TOPIC_V0), "Topics to fetch offsets."));
+            TOPICS_V0);
 
+    // V1 begins support for fetching offsets from the internal __consumer_offsets topic
     private static final Schema OFFSET_FETCH_REQUEST_V1 = OFFSET_FETCH_REQUEST_V0;
 
+    // V2 adds top-level error code to the response as well as allowing a null offset array to indicate fetch
+    // of all committed offsets for a group
+    private static final Field TOPICS_V2 = TOPICS.nullableWithFields(
+            TOPIC_NAME,
+            PARTITIONS_V0);
     private static final Schema OFFSET_FETCH_REQUEST_V2 = new Schema(
             GROUP_ID,
-            new Field(TOPICS_KEY_NAME, ArrayOf.nullable(OFFSET_FETCH_REQUEST_TOPIC_V0), "Topics to fetch offsets. If the " +
-                    "topic array is null fetch offsets for all topics."));
+            TOPICS_V2);
 
-    /* v3 request is the same as v2. Throttle time has been added to v3 response */
+    // V3 request is the same as v2. Throttle time has been added to v3 response
     private static final Schema OFFSET_FETCH_REQUEST_V3 = OFFSET_FETCH_REQUEST_V2;
 
+    // V4 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema OFFSET_FETCH_REQUEST_V4 = OFFSET_FETCH_REQUEST_V3;
 
+    // V5 adds the leader epoch of the committed offset in the response
+    private static final Schema OFFSET_FETCH_REQUEST_V5 = OFFSET_FETCH_REQUEST_V4;
+
     public static Schema[] schemaVersions() {
         return new Schema[] {OFFSET_FETCH_REQUEST_V0, OFFSET_FETCH_REQUEST_V1, OFFSET_FETCH_REQUEST_V2,
-            OFFSET_FETCH_REQUEST_V3, OFFSET_FETCH_REQUEST_V4};
+            OFFSET_FETCH_REQUEST_V3, OFFSET_FETCH_REQUEST_V4, OFFSET_FETCH_REQUEST_V5};
     }
 
     public static class Builder extends AbstractRequest.Builder {
@@ -128,23 +140,22 @@ public static OffsetFetchRequest forAllPartitions(String groupId) {
         return new OffsetFetchRequest.Builder(groupId, null).build((short) 2);
     }
 
-    // v0, v1, and v2 have the same fields.
     private OffsetFetchRequest(String groupId, List partitions, short version) {
-        super(version);
+        super(ApiKeys.OFFSET_FETCH, version);
         this.groupId = groupId;
         this.partitions = partitions;
     }
 
     public OffsetFetchRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.OFFSET_FETCH, version);
 
-        Object[] topicArray = struct.getArray(TOPICS_KEY_NAME);
+        Object[] topicArray = struct.get(TOPICS);
         if (topicArray != null) {
             partitions = new ArrayList<>();
-            for (Object topicResponseObj : struct.getArray(TOPICS_KEY_NAME)) {
+            for (Object topicResponseObj : topicArray) {
                 Struct topicResponse = (Struct) topicResponseObj;
                 String topic = topicResponse.get(TOPIC_NAME);
-                for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
+                for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
                     Struct partitionResponse = (Struct) partitionResponseObj;
                     int partition = partitionResponse.get(PARTITION_ID);
                     partitions.add(new TopicPartition(topic, partition));
@@ -166,12 +177,14 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
 
         Map responsePartitions = new HashMap<>();
         if (versionId < 2) {
-            for (TopicPartition partition : this.partitions) {
-                responsePartitions.put(partition, new OffsetFetchResponse.PartitionData(
-                        OffsetFetchResponse.INVALID_OFFSET,
-                        OffsetFetchResponse.NO_METADATA,
-                        error));
-            }
+            OffsetFetchResponse.PartitionData partitionError = new OffsetFetchResponse.PartitionData(
+                    OffsetFetchResponse.INVALID_OFFSET,
+                    Optional.empty(),
+                    OffsetFetchResponse.NO_METADATA,
+                    error);
+
+            for (TopicPartition partition : this.partitions)
+                responsePartitions.put(partition, partitionError);
         }
 
         switch (versionId) {
@@ -181,6 +194,7 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
                 return new OffsetFetchResponse(error, responsePartitions);
             case 3:
             case 4:
+            case 5:
                 return new OffsetFetchResponse(throttleTimeMs, error, responsePartitions);
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
@@ -214,25 +228,26 @@ protected Struct toStruct() {
         Struct struct = new Struct(ApiKeys.OFFSET_FETCH.requestSchema(version()));
         struct.set(GROUP_ID, groupId);
         if (partitions != null) {
-            Map> topicsData = CollectionUtils.groupDataByTopic(partitions);
+            Map> topicsData = CollectionUtils.groupPartitionsByTopic(partitions);
 
             List topicArray = new ArrayList<>();
             for (Map.Entry> entries : topicsData.entrySet()) {
-                Struct topicData = struct.instance(TOPICS_KEY_NAME);
+                Struct topicData = struct.instance(TOPICS);
                 topicData.set(TOPIC_NAME, entries.getKey());
                 List partitionArray = new ArrayList<>();
                 for (Integer partitionId : entries.getValue()) {
-                    Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+                    Struct partitionData = topicData.instance(PARTITIONS);
                     partitionData.set(PARTITION_ID, partitionId);
                     partitionArray.add(partitionData);
                 }
-                topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+                topicData.set(PARTITIONS, partitionArray.toArray());
                 topicArray.add(topicData);
             }
-            struct.set(TOPICS_KEY_NAME, topicArray.toArray());
+            struct.set(TOPICS, topicArray.toArray());
         } else
-            struct.set(TOPICS_KEY_NAME, null);
+            struct.set(TOPICS, null);
 
         return struct;
     }
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
index 613695bf09d10..2022bb7791089 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -32,79 +31,94 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_LEADER_EPOCH;
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_METADATA;
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_OFFSET;
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
 
+/**
+ * Possible error codes:
+ *
+ * - Partition errors:
+ *   - UNKNOWN_TOPIC_OR_PARTITION (3)
+ *
+ * - Group or coordinator errors:
+ *   - COORDINATOR_LOAD_IN_PROGRESS (14)
+ *   - COORDINATOR_NOT_AVAILABLE (15)
+ *   - NOT_COORDINATOR (16)
+ *   - GROUP_AUTHORIZATION_FAILED (30)
+ */
 public class OffsetFetchResponse extends AbstractResponse {
-
-    private static final String RESPONSES_KEY_NAME = "responses";
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("responses",
+            "Responses by topic for fetched offsets");
 
     // topic level fields
-    private static final String PARTITIONS_KEY_NAME = "partition_responses";
-
-    // partition level fields
-    private static final String COMMIT_OFFSET_KEY_NAME = "offset";
-    private static final String METADATA_KEY_NAME = "metadata";
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partition_responses",
+            "Responses by partition for fetched offsets");
 
-    private static final Schema OFFSET_FETCH_RESPONSE_PARTITION_V0 = new Schema(
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
-            new Field(COMMIT_OFFSET_KEY_NAME, INT64, "Last committed message offset."),
-            new Field(METADATA_KEY_NAME, NULLABLE_STRING, "Any associated metadata the client wants to keep."),
+            COMMITTED_OFFSET,
+            COMMITTED_METADATA,
             ERROR_CODE);
 
-    private static final Schema OFFSET_FETCH_RESPONSE_TOPIC_V0 = new Schema(
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_FETCH_RESPONSE_PARTITION_V0)));
+            PARTITIONS_V0);
 
     private static final Schema OFFSET_FETCH_RESPONSE_V0 = new Schema(
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(OFFSET_FETCH_RESPONSE_TOPIC_V0)));
+            TOPICS_V0);
 
+    // V1 begins support for fetching offsets from the internal __consumer_offsets topic
     private static final Schema OFFSET_FETCH_RESPONSE_V1 = OFFSET_FETCH_RESPONSE_V0;
 
+    // V2 adds top-level error code
     private static final Schema OFFSET_FETCH_RESPONSE_V2 = new Schema(
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(OFFSET_FETCH_RESPONSE_TOPIC_V0)),
+            TOPICS_V0,
             ERROR_CODE);
 
-    /* v3 request is the same as v2. Throttle time has been added to v3 response */
+    // V3 request includes throttle time
     private static final Schema OFFSET_FETCH_RESPONSE_V3 = new Schema(
             THROTTLE_TIME_MS,
-            new Field(RESPONSES_KEY_NAME, new ArrayOf(OFFSET_FETCH_RESPONSE_TOPIC_V0)),
+            TOPICS_V0,
             ERROR_CODE);
 
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+    // V4 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema OFFSET_FETCH_RESPONSE_V4 = OFFSET_FETCH_RESPONSE_V3;
 
+    // V5 adds the leader epoch to the committed offset
+    private static final Field PARTITIONS_V5 = PARTITIONS.withFields(
+            PARTITION_ID,
+            COMMITTED_OFFSET,
+            COMMITTED_LEADER_EPOCH,
+            COMMITTED_METADATA,
+            ERROR_CODE);
+
+    private static final Field TOPICS_V5 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V5);
+
+    private static final Schema OFFSET_FETCH_RESPONSE_V5 = new Schema(
+            THROTTLE_TIME_MS,
+            TOPICS_V5,
+            ERROR_CODE);
+
     public static Schema[] schemaVersions() {
         return new Schema[] {OFFSET_FETCH_RESPONSE_V0, OFFSET_FETCH_RESPONSE_V1, OFFSET_FETCH_RESPONSE_V2,
-            OFFSET_FETCH_RESPONSE_V3, OFFSET_FETCH_RESPONSE_V4};
+            OFFSET_FETCH_RESPONSE_V3, OFFSET_FETCH_RESPONSE_V4, OFFSET_FETCH_RESPONSE_V5};
     }
 
     public static final long INVALID_OFFSET = -1L;
     public static final String NO_METADATA = "";
-    public static final PartitionData UNKNOWN_PARTITION = new PartitionData(INVALID_OFFSET, NO_METADATA,
-            Errors.UNKNOWN_TOPIC_OR_PARTITION);
-    public static final PartitionData UNAUTHORIZED_PARTITION = new PartitionData(INVALID_OFFSET, NO_METADATA,
-            Errors.TOPIC_AUTHORIZATION_FAILED);
-
-    /**
-     * Possible error codes:
-     *
-     * - Partition errors:
-     *   - UNKNOWN_TOPIC_OR_PARTITION (3)
-     *
-     * - Group or coordinator errors:
-     *   - COORDINATOR_LOAD_IN_PROGRESS (14)
-     *   - COORDINATOR_NOT_AVAILABLE (15)
-     *   - NOT_COORDINATOR (16)
-     *   - GROUP_AUTHORIZATION_FAILED (30)
-     */
+    public static final PartitionData UNKNOWN_PARTITION = new PartitionData(INVALID_OFFSET,
+            Optional.empty(), NO_METADATA, Errors.UNKNOWN_TOPIC_OR_PARTITION);
+    public static final PartitionData UNAUTHORIZED_PARTITION = new PartitionData(INVALID_OFFSET,
+            Optional.empty(), NO_METADATA, Errors.TOPIC_AUTHORIZATION_FAILED);
 
     private static final List PARTITION_ERRORS = Collections.singletonList(Errors.UNKNOWN_TOPIC_OR_PARTITION);
 
@@ -116,9 +130,14 @@ public static final class PartitionData {
         public final long offset;
         public final String metadata;
         public final Errors error;
+        public final Optional leaderEpoch;
 
-        public PartitionData(long offset, String metadata, Errors error) {
+        public PartitionData(long offset,
+                             Optional leaderEpoch,
+                             String metadata,
+                             Errors error) {
             this.offset = offset;
+            this.leaderEpoch = leaderEpoch;
             this.metadata = metadata;
             this.error = error;
         }
@@ -153,18 +172,21 @@ public OffsetFetchResponse(Struct struct) {
         this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME);
         Errors topLevelError = Errors.NONE;
         this.responseData = new HashMap<>();
-        for (Object topicResponseObj : struct.getArray(RESPONSES_KEY_NAME)) {
+        for (Object topicResponseObj : struct.get(TOPICS)) {
             Struct topicResponse = (Struct) topicResponseObj;
             String topic = topicResponse.get(TOPIC_NAME);
-            for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionResponseObj : topicResponse.get(PARTITIONS)) {
                 Struct partitionResponse = (Struct) partitionResponseObj;
                 int partition = partitionResponse.get(PARTITION_ID);
-                long offset = partitionResponse.getLong(COMMIT_OFFSET_KEY_NAME);
-                String metadata = partitionResponse.getString(METADATA_KEY_NAME);
+                long offset = partitionResponse.get(COMMITTED_OFFSET);
+                String metadata = partitionResponse.get(COMMITTED_METADATA);
+                Optional leaderEpochOpt = RequestUtils.getLeaderEpoch(partitionResponse, COMMITTED_LEADER_EPOCH);
+
                 Errors error = Errors.forCode(partitionResponse.get(ERROR_CODE));
                 if (error != Errors.NONE && !PARTITION_ERRORS.contains(error))
                     topLevelError = error;
-                PartitionData partitionData = new PartitionData(offset, metadata, error);
+
+                PartitionData partitionData = new PartitionData(offset, leaderEpochOpt, metadata, error);
                 this.responseData.put(new TopicPartition(topic, partition), partitionData);
             }
         }
@@ -215,25 +237,26 @@ protected Struct toStruct(short version) {
         Struct struct = new Struct(ApiKeys.OFFSET_FETCH.responseSchema(version));
         struct.setIfExists(THROTTLE_TIME_MS, throttleTimeMs);
 
-        Map> topicsData = CollectionUtils.groupDataByTopic(responseData);
+        Map> topicsData = CollectionUtils.groupPartitionDataByTopic(responseData);
         List topicArray = new ArrayList<>();
         for (Map.Entry> entries : topicsData.entrySet()) {
-            Struct topicData = struct.instance(RESPONSES_KEY_NAME);
+            Struct topicData = struct.instance(TOPICS);
             topicData.set(TOPIC_NAME, entries.getKey());
             List partitionArray = new ArrayList<>();
             for (Map.Entry partitionEntry : entries.getValue().entrySet()) {
                 PartitionData fetchPartitionData = partitionEntry.getValue();
-                Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+                Struct partitionData = topicData.instance(PARTITIONS);
                 partitionData.set(PARTITION_ID, partitionEntry.getKey());
-                partitionData.set(COMMIT_OFFSET_KEY_NAME, fetchPartitionData.offset);
-                partitionData.set(METADATA_KEY_NAME, fetchPartitionData.metadata);
+                partitionData.set(COMMITTED_OFFSET, fetchPartitionData.offset);
+                RequestUtils.setLeaderEpochIfExists(partitionData, COMMITTED_LEADER_EPOCH, fetchPartitionData.leaderEpoch);
+                partitionData.set(COMMITTED_METADATA, fetchPartitionData.metadata);
                 partitionData.set(ERROR_CODE, fetchPartitionData.error.code());
                 partitionArray.add(partitionData);
             }
-            topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+            topicData.set(PARTITIONS, partitionArray.toArray());
             topicArray.add(topicData);
         }
-        struct.set(RESPONSES_KEY_NAME, topicArray.toArray());
+        struct.set(TOPICS, topicArray.toArray());
 
         if (version > 1)
             struct.set(ERROR_CODE, this.error.code());
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java
index 651416d97a7a7..9de5d02d4afa9 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -30,53 +29,69 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
+import static org.apache.kafka.common.protocol.CommonFields.CURRENT_LEADER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
 
 public class OffsetsForLeaderEpochRequest extends AbstractRequest {
-    private static final String TOPICS_KEY_NAME = "topics";
-    private static final String PARTITIONS_KEY_NAME = "partitions";
-    private static final String LEADER_EPOCH = "leader_epoch";
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "An array of topics to get epochs for");
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "An array of partitions to get epochs for");
 
-    /* Offsets for Leader Epoch api */
-    private static final Schema OFFSET_FOR_LEADER_EPOCH_REQUEST_PARTITION_V0 = new Schema(
+    private static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch",
+            "The epoch to lookup an offset for.");
+
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
-            new Field(LEADER_EPOCH, INT32, "The epoch"));
-    private static final Schema OFFSET_FOR_LEADER_EPOCH_REQUEST_TOPIC_V0 = new Schema(
+            LEADER_EPOCH);
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_FOR_LEADER_EPOCH_REQUEST_PARTITION_V0)));
+            PARTITIONS_V0);
     private static final Schema OFFSET_FOR_LEADER_EPOCH_REQUEST_V0 = new Schema(
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_FOR_LEADER_EPOCH_REQUEST_TOPIC_V0), "An array of topics to get epochs for"));
+            TOPICS_V0);
 
-    /* v1 request is the same as v0. Per-partition leader epoch has been added to response */
+    // V1 request is the same as v0. Per-partition leader epoch has been added to response
     private static final Schema OFFSET_FOR_LEADER_EPOCH_REQUEST_V1 = OFFSET_FOR_LEADER_EPOCH_REQUEST_V0;
 
+    // V2 adds the current leader epoch to support fencing
+    private static final Field PARTITIONS_V2 = PARTITIONS.withFields(
+            PARTITION_ID,
+            CURRENT_LEADER_EPOCH,
+            LEADER_EPOCH);
+    private static final Field TOPICS_V2 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V2);
+    private static final Schema OFFSET_FOR_LEADER_EPOCH_REQUEST_V2 = new Schema(
+            TOPICS_V2);
+
     public static Schema[] schemaVersions() {
-        return new Schema[]{OFFSET_FOR_LEADER_EPOCH_REQUEST_V0, OFFSET_FOR_LEADER_EPOCH_REQUEST_V1};
+        return new Schema[]{OFFSET_FOR_LEADER_EPOCH_REQUEST_V0, OFFSET_FOR_LEADER_EPOCH_REQUEST_V1,
+            OFFSET_FOR_LEADER_EPOCH_REQUEST_V2};
     }
 
-    private Map epochsByPartition;
+    private Map epochsByPartition;
 
-    public Map epochsByTopicPartition() {
+    public Map epochsByTopicPartition() {
         return epochsByPartition;
     }
 
     public static class Builder extends AbstractRequest.Builder {
-        private Map epochsByPartition = new HashMap<>();
+        private Map epochsByPartition;
 
         public Builder(short version) {
-            super(ApiKeys.OFFSET_FOR_LEADER_EPOCH, version);
+            this(version, new HashMap<>());
         }
 
-        public Builder(short version, Map epochsByPartition) {
+        public Builder(short version, Map epochsByPartition) {
             super(ApiKeys.OFFSET_FOR_LEADER_EPOCH, version);
             this.epochsByPartition = epochsByPartition;
         }
 
-        public Builder add(TopicPartition topicPartition, Integer epoch) {
-            epochsByPartition.put(topicPartition, epoch);
+        public Builder add(TopicPartition topicPartition, Optional currentEpoch, int leaderEpoch) {
+            epochsByPartition.put(topicPartition, new PartitionData(currentEpoch, leaderEpoch));
             return this;
         }
 
@@ -99,23 +114,24 @@ public String toString() {
         }
     }
 
-    public OffsetsForLeaderEpochRequest(Map epochsByPartition, short version) {
-        super(version);
+    public OffsetsForLeaderEpochRequest(Map epochsByPartition, short version) {
+        super(ApiKeys.OFFSET_FOR_LEADER_EPOCH, version);
         this.epochsByPartition = epochsByPartition;
     }
 
     public OffsetsForLeaderEpochRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.OFFSET_FOR_LEADER_EPOCH, version);
         epochsByPartition = new HashMap<>();
-        for (Object topicAndEpochsObj : struct.getArray(TOPICS_KEY_NAME)) {
+        for (Object topicAndEpochsObj : struct.get(TOPICS)) {
             Struct topicAndEpochs = (Struct) topicAndEpochsObj;
             String topic = topicAndEpochs.get(TOPIC_NAME);
-            for (Object partitionAndEpochObj : topicAndEpochs.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionAndEpochObj : topicAndEpochs.get(PARTITIONS)) {
                 Struct partitionAndEpoch = (Struct) partitionAndEpochObj;
                 int partitionId = partitionAndEpoch.get(PARTITION_ID);
-                int epoch = partitionAndEpoch.getInt(LEADER_EPOCH);
+                int leaderEpoch = partitionAndEpoch.get(LEADER_EPOCH);
+                Optional currentEpoch = RequestUtils.getLeaderEpoch(partitionAndEpoch, CURRENT_LEADER_EPOCH);
                 TopicPartition tp = new TopicPartition(topic, partitionId);
-                epochsByPartition.put(tp, epoch);
+                epochsByPartition.put(tp, new PartitionData(currentEpoch, leaderEpoch));
             }
         }
     }
@@ -128,23 +144,28 @@ public static OffsetsForLeaderEpochRequest parse(ByteBuffer buffer, short versio
     protected Struct toStruct() {
         Struct requestStruct = new Struct(ApiKeys.OFFSET_FOR_LEADER_EPOCH.requestSchema(version()));
 
-        Map> topicsToPartitionEpochs = CollectionUtils.groupDataByTopic(epochsByPartition);
+        Map> topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(epochsByPartition);
 
         List topics = new ArrayList<>();
-        for (Map.Entry> topicToEpochs : topicsToPartitionEpochs.entrySet()) {
-            Struct topicsStruct = requestStruct.instance(TOPICS_KEY_NAME);
+        for (Map.Entry> topicToEpochs : topicsToPartitionEpochs.entrySet()) {
+            Struct topicsStruct = requestStruct.instance(TOPICS);
             topicsStruct.set(TOPIC_NAME, topicToEpochs.getKey());
             List partitions = new ArrayList<>();
-            for (Map.Entry partitionEpoch : topicToEpochs.getValue().entrySet()) {
-                Struct partitionStruct = topicsStruct.instance(PARTITIONS_KEY_NAME);
+            for (Map.Entry partitionEpoch : topicToEpochs.getValue().entrySet()) {
+                Struct partitionStruct = topicsStruct.instance(PARTITIONS);
                 partitionStruct.set(PARTITION_ID, partitionEpoch.getKey());
-                partitionStruct.set(LEADER_EPOCH, partitionEpoch.getValue());
+
+                PartitionData partitionData = partitionEpoch.getValue();
+                partitionStruct.set(LEADER_EPOCH, partitionData.leaderEpoch);
+
+                // Current leader epoch introduced in v2
+                RequestUtils.setLeaderEpochIfExists(partitionStruct, CURRENT_LEADER_EPOCH, partitionData.currentLeaderEpoch);
                 partitions.add(partitionStruct);
             }
-            topicsStruct.set(PARTITIONS_KEY_NAME, partitions.toArray());
+            topicsStruct.set(PARTITIONS, partitions.toArray());
             topics.add(topicsStruct);
         }
-        requestStruct.set(TOPICS_KEY_NAME, topics.toArray());
+        requestStruct.set(TOPICS, topics.toArray());
         return requestStruct;
     }
 
@@ -158,4 +179,15 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         }
         return new OffsetsForLeaderEpochResponse(errorResponse);
     }
+
+    public static class PartitionData {
+        public final Optional currentLeaderEpoch;
+        public final int leaderEpoch;
+
+        public PartitionData(Optional currentLeaderEpoch, int leaderEpoch) {
+            this.currentLeaderEpoch = currentLeaderEpoch;
+            this.leaderEpoch = leaderEpoch;
+        }
+
+    }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java
index 4da876704b7af..324a2ed5f232a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -33,59 +32,61 @@
 import java.util.Map;
 
 import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
+import static org.apache.kafka.common.protocol.CommonFields.LEADER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.CommonFields.LEADER_EPOCH;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
 
 public class OffsetsForLeaderEpochResponse extends AbstractResponse {
-    private static final String TOPICS_KEY_NAME = "topics";
-    private static final String PARTITIONS_KEY_NAME = "partitions";
-    private static final String END_OFFSET_KEY_NAME = "end_offset";
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "An array of topics for which we have leader offsets for some requested partition leader epoch");
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "An array of offsets by partition");
+    private static final Field.Int64 END_OFFSET = new Field.Int64("end_offset", "The end offset");
 
-    private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_PARTITION_V0 = new Schema(
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             ERROR_CODE,
             PARTITION_ID,
-            new Field(END_OFFSET_KEY_NAME, INT64, "The end offset"));
-    private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_TOPIC_V0 = new Schema(
+            END_OFFSET);
+    private static final Field TOPICS_V0 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_FOR_LEADER_EPOCH_RESPONSE_PARTITION_V0)));
+            PARTITIONS_V0);
     private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_V0 = new Schema(
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_FOR_LEADER_EPOCH_RESPONSE_TOPIC_V0),
-                    "An array of topics for which we have leader offsets for some requested Partition Leader Epoch"));
+            TOPICS_V0);
 
-    // OFFSET_FOR_LEADER_EPOCH_RESPONSE_PARTITION_V1 added a per-partition leader epoch field,
-    // which specifies which leader epoch the end offset belongs to
-    private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_PARTITION_V1 = new Schema(
+    // V1 added a per-partition leader epoch field which specifies which leader epoch the end offset belongs to
+    private static final Field PARTITIONS_V1 = PARTITIONS.withFields(
             ERROR_CODE,
             PARTITION_ID,
             LEADER_EPOCH,
-            new Field(END_OFFSET_KEY_NAME, INT64, "The end offset"));
-    private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_TOPIC_V1 = new Schema(
+            END_OFFSET);
+    private static final Field TOPICS_V1 = TOPICS.withFields(
             TOPIC_NAME,
-            new Field(PARTITIONS_KEY_NAME, new ArrayOf(OFFSET_FOR_LEADER_EPOCH_RESPONSE_PARTITION_V1)));
+            PARTITIONS_V1);
     private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_V1 = new Schema(
-            new Field(TOPICS_KEY_NAME, new ArrayOf(OFFSET_FOR_LEADER_EPOCH_RESPONSE_TOPIC_V1),
-                  "An array of topics for which we have leader offsets for some requested Partition Leader Epoch"));
+            TOPICS_V1);
+
+    // V2 bumped for addition of current leader epoch to the request schema.
+    private static final Schema OFFSET_FOR_LEADER_EPOCH_RESPONSE_V2 = OFFSET_FOR_LEADER_EPOCH_RESPONSE_V1;
 
     public static Schema[] schemaVersions() {
-        return new Schema[]{OFFSET_FOR_LEADER_EPOCH_RESPONSE_V0, OFFSET_FOR_LEADER_EPOCH_RESPONSE_V1};
+        return new Schema[]{OFFSET_FOR_LEADER_EPOCH_RESPONSE_V0, OFFSET_FOR_LEADER_EPOCH_RESPONSE_V1,
+            OFFSET_FOR_LEADER_EPOCH_RESPONSE_V2};
     }
 
     private Map epochEndOffsetsByPartition;
 
     public OffsetsForLeaderEpochResponse(Struct struct) {
         epochEndOffsetsByPartition = new HashMap<>();
-        for (Object topicAndEpocsObj : struct.getArray(TOPICS_KEY_NAME)) {
+        for (Object topicAndEpocsObj : struct.get(TOPICS)) {
             Struct topicAndEpochs = (Struct) topicAndEpocsObj;
             String topic = topicAndEpochs.get(TOPIC_NAME);
-            for (Object partitionAndEpochObj : topicAndEpochs.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionAndEpochObj : topicAndEpochs.get(PARTITIONS)) {
                 Struct partitionAndEpoch = (Struct) partitionAndEpochObj;
                 Errors error = Errors.forCode(partitionAndEpoch.get(ERROR_CODE));
                 int partitionId = partitionAndEpoch.get(PARTITION_ID);
                 TopicPartition tp = new TopicPartition(topic, partitionId);
                 int leaderEpoch = partitionAndEpoch.getOrElse(LEADER_EPOCH, RecordBatch.NO_PARTITION_LEADER_EPOCH);
-                long endOffset = partitionAndEpoch.getLong(END_OFFSET_KEY_NAME);
+                long endOffset = partitionAndEpoch.get(END_OFFSET);
                 epochEndOffsetsByPartition.put(tp, new EpochEndOffset(error, leaderEpoch, endOffset));
             }
         }
@@ -115,26 +116,26 @@ public static OffsetsForLeaderEpochResponse parse(ByteBuffer buffer, short versi
     protected Struct toStruct(short version) {
         Struct responseStruct = new Struct(ApiKeys.OFFSET_FOR_LEADER_EPOCH.responseSchema(version));
 
-        Map> endOffsetsByTopic = CollectionUtils.groupDataByTopic(epochEndOffsetsByPartition);
+        Map> endOffsetsByTopic = CollectionUtils.groupPartitionDataByTopic(epochEndOffsetsByPartition);
 
         List topics = new ArrayList<>(endOffsetsByTopic.size());
         for (Map.Entry> topicToPartitionEpochs : endOffsetsByTopic.entrySet()) {
-            Struct topicStruct = responseStruct.instance(TOPICS_KEY_NAME);
+            Struct topicStruct = responseStruct.instance(TOPICS);
             topicStruct.set(TOPIC_NAME, topicToPartitionEpochs.getKey());
             Map partitionEpochs = topicToPartitionEpochs.getValue();
             List partitions = new ArrayList<>();
             for (Map.Entry partitionEndOffset : partitionEpochs.entrySet()) {
-                Struct partitionStruct = topicStruct.instance(PARTITIONS_KEY_NAME);
+                Struct partitionStruct = topicStruct.instance(PARTITIONS);
                 partitionStruct.set(ERROR_CODE, partitionEndOffset.getValue().error().code());
                 partitionStruct.set(PARTITION_ID, partitionEndOffset.getKey());
                 partitionStruct.setIfExists(LEADER_EPOCH, partitionEndOffset.getValue().leaderEpoch());
-                partitionStruct.set(END_OFFSET_KEY_NAME, partitionEndOffset.getValue().endOffset());
+                partitionStruct.set(END_OFFSET, partitionEndOffset.getValue().endOffset());
                 partitions.add(partitionStruct);
             }
-            topicStruct.set(PARTITIONS_KEY_NAME, partitions.toArray());
+            topicStruct.set(PARTITIONS, partitions.toArray());
             topics.add(topicStruct);
         }
-        responseStruct.set(TOPICS_KEY_NAME, topics.toArray());
+        responseStruct.set(TOPICS, topics.toArray());
         return responseStruct;
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
index 67745cbb4cc7a..4f1d766b8bf3e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
@@ -196,7 +196,7 @@ public String toString() {
     private boolean idempotent = false;
 
     private ProduceRequest(short version, short acks, int timeout, Map partitionRecords, String transactionalId) {
-        super(version);
+        super(ApiKeys.PRODUCE, version);
         this.acks = acks;
         this.timeout = timeout;
 
@@ -216,7 +216,7 @@ private static Map createPartitionSizes(Map();
         for (Object topicDataObj : struct.getArray(TOPIC_DATA_KEY_NAME)) {
             Struct topicData = (Struct) topicDataObj;
@@ -268,7 +268,7 @@ public Struct toStruct() {
         Map partitionRecords = partitionRecordsOrFail();
         short version = version();
         Struct struct = new Struct(ApiKeys.PRODUCE.requestSchema(version));
-        Map> recordsByTopic = CollectionUtils.groupDataByTopic(partitionRecords);
+        Map> recordsByTopic = CollectionUtils.groupPartitionDataByTopic(partitionRecords);
         struct.set(ACKS_KEY_NAME, acks);
         struct.set(TIMEOUT_KEY_NAME, timeout);
         struct.setIfExists(NULLABLE_TRANSACTIONAL_ID, transactionalId);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
index 467c9804c3c40..fb15813ccf1eb 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
@@ -196,7 +196,7 @@ public ProduceResponse(Struct struct) {
     protected Struct toStruct(short version) {
         Struct struct = new Struct(ApiKeys.PRODUCE.responseSchema(version));
 
-        Map> responseByTopic = CollectionUtils.groupDataByTopic(responses);
+        Map> responseByTopic = CollectionUtils.groupPartitionDataByTopic(responses);
         List topicDatas = new ArrayList<>(responseByTopic.size());
         for (Map.Entry> entry : responseByTopic.entrySet()) {
             Struct topicData = struct.instance(RESPONSES_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java
index 57c310014f524..d73561abbb88e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java
@@ -44,14 +44,14 @@ public class RenewDelegationTokenRequest extends AbstractRequest {
     public static final Schema TOKEN_RENEW_REQUEST_V1 = TOKEN_RENEW_REQUEST_V0;
 
     private RenewDelegationTokenRequest(short version, ByteBuffer hmac, long renewTimePeriod) {
-        super(version);
+        super(ApiKeys.RENEW_DELEGATION_TOKEN, version);
 
         this.hmac = hmac;
         this.renewTimePeriod = renewTimePeriod;
     }
 
     public RenewDelegationTokenRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.RENEW_DELEGATION_TOKEN, versionId);
 
         hmac = struct.getBytes(HMAC_KEY_NAME);
         renewTimePeriod = struct.getLong(RENEW_TIME_PERIOD_KEY_NAME);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
index 7638c6c20a166..24c2fbe441661 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java
@@ -20,12 +20,16 @@
 import org.apache.kafka.common.acl.AccessControlEntryFilter;
 import org.apache.kafka.common.acl.AclOperation;
 import org.apache.kafka.common.acl.AclPermissionType;
+import org.apache.kafka.common.protocol.types.Field;
+import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.resource.PatternType;
 import org.apache.kafka.common.resource.ResourcePattern;
 import org.apache.kafka.common.resource.ResourcePatternFilter;
 import org.apache.kafka.common.protocol.types.Struct;
 import org.apache.kafka.common.resource.ResourceType;
 
+import java.util.Optional;
+
 import static org.apache.kafka.common.protocol.CommonFields.HOST;
 import static org.apache.kafka.common.protocol.CommonFields.HOST_FILTER;
 import static org.apache.kafka.common.protocol.CommonFields.OPERATION;
@@ -101,4 +105,16 @@ static void aceFilterSetStructFields(AccessControlEntryFilter filter, Struct str
         struct.set(OPERATION, filter.operation().code());
         struct.set(PERMISSION_TYPE, filter.permissionType().code());
     }
+
+    static void setLeaderEpochIfExists(Struct struct, Field.Int32 leaderEpochField, Optional leaderEpoch) {
+        struct.setIfExists(leaderEpochField, leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH));
+    }
+
+    static Optional getLeaderEpoch(Struct struct, Field.Int32 leaderEpochField) {
+        int leaderEpoch = struct.getOrElse(leaderEpochField, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+        Optional leaderEpochOpt = leaderEpoch == RecordBatch.NO_PARTITION_LEADER_EPOCH ?
+                Optional.empty() : Optional.of(leaderEpoch);
+        return leaderEpochOpt;
+    }
+
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java
index 74d31a688de66..2ce144e59205e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java
@@ -72,12 +72,12 @@ public SaslAuthenticateRequest(ByteBuffer saslAuthBytes) {
     }
 
     public SaslAuthenticateRequest(ByteBuffer saslAuthBytes, short version) {
-        super(version);
+        super(ApiKeys.SASL_AUTHENTICATE, version);
         this.saslAuthBytes = saslAuthBytes;
     }
 
     public SaslAuthenticateRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.SASL_AUTHENTICATE, version);
         saslAuthBytes = struct.getBytes(SASL_AUTH_BYTES_KEY_NAME);
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
index a06a4db870ce1..7225eb7b8f807 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
@@ -80,12 +80,12 @@ public SaslHandshakeRequest(String mechanism) {
     }
 
     public SaslHandshakeRequest(String mechanism, short version) {
-        super(version);
+        super(ApiKeys.SASL_HANDSHAKE, version);
         this.mechanism = mechanism;
     }
 
     public SaslHandshakeRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.SASL_HANDSHAKE, version);
         mechanism = struct.getString(MECHANISM_KEY_NAME);
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index d79c938d7941b..a296c8059a8c7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -98,7 +98,7 @@ public String toString() {
 
     private StopReplicaRequest(int controllerId, int controllerEpoch, boolean deletePartitions,
                                Set partitions, short version) {
-        super(version);
+        super(ApiKeys.STOP_REPLICA, version);
         this.controllerId = controllerId;
         this.controllerEpoch = controllerEpoch;
         this.deletePartitions = deletePartitions;
@@ -106,7 +106,7 @@ private StopReplicaRequest(int controllerId, int controllerEpoch, boolean delete
     }
 
     public StopReplicaRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.STOP_REPLICA, version);
 
         partitions = new HashSet<>();
         for (Object partitionDataObj : struct.getArray(PARTITIONS_KEY_NAME)) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
index 962bc77884dcb..237320f9cd25e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
@@ -101,7 +101,7 @@ public String toString() {
 
     private SyncGroupRequest(String groupId, int generationId, String memberId,
                              Map groupAssignment, short version) {
-        super(version);
+        super(ApiKeys.SYNC_GROUP, version);
         this.groupId = groupId;
         this.generationId = generationId;
         this.memberId = memberId;
@@ -109,7 +109,7 @@ private SyncGroupRequest(String groupId, int generationId, String memberId,
     }
 
     public SyncGroupRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.SYNC_GROUP, version);
         this.groupId = struct.get(GROUP_ID);
         this.generationId = struct.get(GENERATION_ID);
         this.memberId = struct.get(MEMBER_ID);
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java
index 25245be814f8d..1c922e1dd525c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -28,44 +27,66 @@
 import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_LEADER_EPOCH;
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_METADATA;
+import static org.apache.kafka.common.protocol.CommonFields.COMMITTED_OFFSET;
 import static org.apache.kafka.common.protocol.CommonFields.GROUP_ID;
 import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
 import static org.apache.kafka.common.protocol.CommonFields.PRODUCER_EPOCH;
 import static org.apache.kafka.common.protocol.CommonFields.PRODUCER_ID;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
 import static org.apache.kafka.common.protocol.CommonFields.TRANSACTIONAL_ID;
-import static org.apache.kafka.common.protocol.types.Type.INT64;
-import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
 
 public class TxnOffsetCommitRequest extends AbstractRequest {
-    private static final String TOPICS_KEY_NAME = "topics";
-    private static final String PARTITIONS_KEY_NAME = "partitions";
-    private static final String OFFSET_KEY_NAME = "offset";
-    private static final String METADATA_KEY_NAME = "metadata";
+    // top level fields
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "Topics to commit offsets");
 
-    private static final Schema TXN_OFFSET_COMMIT_PARTITION_OFFSET_METADATA_REQUEST_V0 = new Schema(
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "Partitions to commit offsets");
+
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
-            new Field(OFFSET_KEY_NAME, INT64),
-            new Field(METADATA_KEY_NAME, NULLABLE_STRING));
+            COMMITTED_OFFSET,
+            COMMITTED_METADATA);
+
+    private static final Field TOPICS_V0 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V0);
 
     private static final Schema TXN_OFFSET_COMMIT_REQUEST_V0 = new Schema(
             TRANSACTIONAL_ID,
             GROUP_ID,
             PRODUCER_ID,
             PRODUCER_EPOCH,
-            new Field(TOPICS_KEY_NAME, new ArrayOf(new Schema(
-                    TOPIC_NAME,
-                    new Field(PARTITIONS_KEY_NAME, new ArrayOf(TXN_OFFSET_COMMIT_PARTITION_OFFSET_METADATA_REQUEST_V0)))),
-                    "The partitions to write markers for."));
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+            TOPICS_V0);
+
+    // V1 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema TXN_OFFSET_COMMIT_REQUEST_V1 = TXN_OFFSET_COMMIT_REQUEST_V0;
 
+    // V2 adds the leader epoch to the partition data
+    private static final Field PARTITIONS_V2 = PARTITIONS.withFields(
+            PARTITION_ID,
+            COMMITTED_OFFSET,
+            COMMITTED_LEADER_EPOCH,
+            COMMITTED_METADATA);
+
+    private static final Field TOPICS_V2 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V2);
+
+    private static final Schema TXN_OFFSET_COMMIT_REQUEST_V2 = new Schema(
+            TRANSACTIONAL_ID,
+            GROUP_ID,
+            PRODUCER_ID,
+            PRODUCER_EPOCH,
+            TOPICS_V2);
+
     public static Schema[] schemaVersions() {
-        return new Schema[]{TXN_OFFSET_COMMIT_REQUEST_V0, TXN_OFFSET_COMMIT_REQUEST_V1};
+        return new Schema[]{TXN_OFFSET_COMMIT_REQUEST_V0, TXN_OFFSET_COMMIT_REQUEST_V1, TXN_OFFSET_COMMIT_REQUEST_V2};
     }
 
     public static class Builder extends AbstractRequest.Builder {
@@ -120,7 +141,7 @@ public String toString() {
 
     public TxnOffsetCommitRequest(short version, String transactionalId, String consumerGroupId, long producerId,
                                   short producerEpoch, Map offsets) {
-        super(version);
+        super(ApiKeys.TXN_OFFSET_COMMIT, version);
         this.transactionalId = transactionalId;
         this.consumerGroupId = consumerGroupId;
         this.producerId = producerId;
@@ -129,23 +150,24 @@ public TxnOffsetCommitRequest(short version, String transactionalId, String cons
     }
 
     public TxnOffsetCommitRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.TXN_OFFSET_COMMIT, version);
         this.transactionalId = struct.get(TRANSACTIONAL_ID);
         this.consumerGroupId = struct.get(GROUP_ID);
         this.producerId = struct.get(PRODUCER_ID);
         this.producerEpoch = struct.get(PRODUCER_EPOCH);
 
         Map offsets = new HashMap<>();
-        Object[] topicPartitionsArray = struct.getArray(TOPICS_KEY_NAME);
+        Object[] topicPartitionsArray = struct.get(TOPICS);
         for (Object topicPartitionObj : topicPartitionsArray) {
             Struct topicPartitionStruct = (Struct) topicPartitionObj;
             String topic = topicPartitionStruct.get(TOPIC_NAME);
-            for (Object partitionObj : topicPartitionStruct.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionObj : topicPartitionStruct.get(PARTITIONS)) {
                 Struct partitionStruct = (Struct) partitionObj;
                 TopicPartition partition = new TopicPartition(topic, partitionStruct.get(PARTITION_ID));
-                long offset = partitionStruct.getLong(OFFSET_KEY_NAME);
-                String metadata = partitionStruct.getString(METADATA_KEY_NAME);
-                offsets.put(partition, new CommittedOffset(offset, metadata));
+                long offset = partitionStruct.get(COMMITTED_OFFSET);
+                String metadata = partitionStruct.get(COMMITTED_METADATA);
+                Optional leaderEpoch = RequestUtils.getLeaderEpoch(partitionStruct, COMMITTED_LEADER_EPOCH);
+                offsets.put(partition, new CommittedOffset(offset, metadata, leaderEpoch));
             }
         }
         this.offsets = offsets;
@@ -179,29 +201,31 @@ protected Struct toStruct() {
         struct.set(PRODUCER_ID, producerId);
         struct.set(PRODUCER_EPOCH, producerEpoch);
 
-        Map> mappedPartitionOffsets = CollectionUtils.groupDataByTopic(offsets);
+        Map> mappedPartitionOffsets = CollectionUtils.groupPartitionDataByTopic(offsets);
         Object[] partitionsArray = new Object[mappedPartitionOffsets.size()];
         int i = 0;
         for (Map.Entry> topicAndPartitions : mappedPartitionOffsets.entrySet()) {
-            Struct topicPartitionsStruct = struct.instance(TOPICS_KEY_NAME);
+            Struct topicPartitionsStruct = struct.instance(TOPICS);
             topicPartitionsStruct.set(TOPIC_NAME, topicAndPartitions.getKey());
 
             Map partitionOffsets = topicAndPartitions.getValue();
             Object[] partitionOffsetsArray = new Object[partitionOffsets.size()];
             int j = 0;
             for (Map.Entry partitionOffset : partitionOffsets.entrySet()) {
-                Struct partitionOffsetStruct = topicPartitionsStruct.instance(PARTITIONS_KEY_NAME);
+                Struct partitionOffsetStruct = topicPartitionsStruct.instance(PARTITIONS);
                 partitionOffsetStruct.set(PARTITION_ID, partitionOffset.getKey());
                 CommittedOffset committedOffset = partitionOffset.getValue();
-                partitionOffsetStruct.set(OFFSET_KEY_NAME, committedOffset.offset);
-                partitionOffsetStruct.set(METADATA_KEY_NAME, committedOffset.metadata);
+                partitionOffsetStruct.set(COMMITTED_OFFSET, committedOffset.offset);
+                partitionOffsetStruct.set(COMMITTED_METADATA, committedOffset.metadata);
+                RequestUtils.setLeaderEpochIfExists(partitionOffsetStruct, COMMITTED_LEADER_EPOCH,
+                        committedOffset.leaderEpoch);
                 partitionOffsetsArray[j++] = partitionOffsetStruct;
             }
-            topicPartitionsStruct.set(PARTITIONS_KEY_NAME, partitionOffsetsArray);
+            topicPartitionsStruct.set(PARTITIONS, partitionOffsetsArray);
             partitionsArray[i++] = topicPartitionsStruct;
         }
 
-        struct.set(TOPICS_KEY_NAME, partitionsArray);
+        struct.set(TOPICS, partitionsArray);
         return struct;
     }
 
@@ -219,28 +243,23 @@ public static TxnOffsetCommitRequest parse(ByteBuffer buffer, short version) {
     }
 
     public static class CommittedOffset {
-        private final long offset;
-        private final String metadata;
+        public final long offset;
+        public final String metadata;
+        public final Optional leaderEpoch;
 
-        public CommittedOffset(long offset, String metadata) {
+        public CommittedOffset(long offset, String metadata, Optional leaderEpoch) {
             this.offset = offset;
             this.metadata = metadata;
+            this.leaderEpoch = leaderEpoch;
         }
 
         @Override
         public String toString() {
             return "CommittedOffset(" +
                     "offset=" + offset +
+                    ", leaderEpoch=" + leaderEpoch +
                     ", metadata='" + metadata + "')";
         }
-
-        public long offset() {
-            return offset;
-        }
-
-        public String metadata() {
-            return metadata;
-        }
     }
 
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
index c34fd40023278..ba8f7d6be2eaf 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
@@ -19,7 +19,6 @@
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.ArrayOf;
 import org.apache.kafka.common.protocol.types.Field;
 import org.apache.kafka.common.protocol.types.Schema;
 import org.apache.kafka.common.protocol.types.Struct;
@@ -34,41 +33,49 @@
 import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS;
 import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
 
+/**
+ *
+ * Possible error codes:
+ *   InvalidProducerEpoch
+ *   NotCoordinator
+ *   CoordinatorNotAvailable
+ *   CoordinatorLoadInProgress
+ *   OffsetMetadataTooLarge
+ *   GroupAuthorizationFailed
+ *   InvalidCommitOffsetSize
+ *   TransactionalIdAuthorizationFailed
+ *   RequestTimedOut
+ */
 public class TxnOffsetCommitResponse extends AbstractResponse {
-    private static final String TOPICS_KEY_NAME = "topics";
-    private static final String PARTITIONS_KEY_NAME = "partitions";
+    private static final Field.ComplexArray TOPICS = new Field.ComplexArray("topics",
+            "Responses by topic for committed offsets");
 
-    private static final Schema TXN_OFFSET_COMMIT_PARTITION_ERROR_RESPONSE_V0 = new Schema(
+    // topic level fields
+    private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions",
+            "Responses by partition for committed offsets");
+
+    private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
             PARTITION_ID,
             ERROR_CODE);
 
+    private static final Field TOPICS_V0 = TOPICS.withFields(
+            TOPIC_NAME,
+            PARTITIONS_V0);
+
     private static final Schema TXN_OFFSET_COMMIT_RESPONSE_V0 = new Schema(
             THROTTLE_TIME_MS,
-            new Field(TOPICS_KEY_NAME, new ArrayOf(new Schema(
-                    TOPIC_NAME,
-                    new Field(PARTITIONS_KEY_NAME, new ArrayOf(TXN_OFFSET_COMMIT_PARTITION_ERROR_RESPONSE_V0)))),
-                    "Errors per partition from writing markers."));
-
-    /**
-     * The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
-     */
+            TOPICS_V0);
+
+    // V1 bump used to indicate that on quota violation brokers send out responses before throttling.
     private static final Schema TXN_OFFSET_COMMIT_RESPONSE_V1 = TXN_OFFSET_COMMIT_RESPONSE_V0;
 
+    // V2 adds the leader epoch to the partition data
+    private static final Schema TXN_OFFSET_COMMIT_RESPONSE_V2 = TXN_OFFSET_COMMIT_RESPONSE_V1;
+
     public static Schema[] schemaVersions() {
-        return new Schema[]{TXN_OFFSET_COMMIT_RESPONSE_V0, TXN_OFFSET_COMMIT_RESPONSE_V1};
+        return new Schema[]{TXN_OFFSET_COMMIT_RESPONSE_V0, TXN_OFFSET_COMMIT_RESPONSE_V1, TXN_OFFSET_COMMIT_RESPONSE_V2};
     }
 
-    // Possible error codes:
-    //   InvalidProducerEpoch
-    //   NotCoordinator
-    //   CoordinatorNotAvailable
-    //   CoordinatorLoadInProgress
-    //   OffsetMetadataTooLarge
-    //   GroupAuthorizationFailed
-    //   InvalidCommitOffsetSize
-    //   TransactionalIdAuthorizationFailed
-    //   RequestTimedOut
-
     private final Map errors;
     private final int throttleTimeMs;
 
@@ -80,11 +87,11 @@ public TxnOffsetCommitResponse(int throttleTimeMs, Map e
     public TxnOffsetCommitResponse(Struct struct) {
         this.throttleTimeMs = struct.get(THROTTLE_TIME_MS);
         Map errors = new HashMap<>();
-        Object[] topicPartitionsArray = struct.getArray(TOPICS_KEY_NAME);
+        Object[] topicPartitionsArray = struct.get(TOPICS);
         for (Object topicPartitionObj : topicPartitionsArray) {
             Struct topicPartitionStruct = (Struct) topicPartitionObj;
             String topic = topicPartitionStruct.get(TOPIC_NAME);
-            for (Object partitionObj : topicPartitionStruct.getArray(PARTITIONS_KEY_NAME)) {
+            for (Object partitionObj : topicPartitionStruct.get(PARTITIONS)) {
                 Struct partitionStruct = (Struct) partitionObj;
                 Integer partition = partitionStruct.get(PARTITION_ID);
                 Errors error = Errors.forCode(partitionStruct.get(ERROR_CODE));
@@ -98,27 +105,27 @@ public TxnOffsetCommitResponse(Struct struct) {
     protected Struct toStruct(short version) {
         Struct struct = new Struct(ApiKeys.TXN_OFFSET_COMMIT.responseSchema(version));
         struct.set(THROTTLE_TIME_MS, throttleTimeMs);
-        Map> mappedPartitions = CollectionUtils.groupDataByTopic(errors);
+        Map> mappedPartitions = CollectionUtils.groupPartitionDataByTopic(errors);
         Object[] partitionsArray = new Object[mappedPartitions.size()];
         int i = 0;
         for (Map.Entry> topicAndPartitions : mappedPartitions.entrySet()) {
-            Struct topicPartitionsStruct = struct.instance(TOPICS_KEY_NAME);
+            Struct topicPartitionsStruct = struct.instance(TOPICS);
             topicPartitionsStruct.set(TOPIC_NAME, topicAndPartitions.getKey());
             Map partitionAndErrors = topicAndPartitions.getValue();
 
             Object[] partitionAndErrorsArray = new Object[partitionAndErrors.size()];
             int j = 0;
             for (Map.Entry partitionAndError : partitionAndErrors.entrySet()) {
-                Struct partitionAndErrorStruct = topicPartitionsStruct.instance(PARTITIONS_KEY_NAME);
+                Struct partitionAndErrorStruct = topicPartitionsStruct.instance(PARTITIONS);
                 partitionAndErrorStruct.set(PARTITION_ID, partitionAndError.getKey());
                 partitionAndErrorStruct.set(ERROR_CODE, partitionAndError.getValue().code());
                 partitionAndErrorsArray[j++] = partitionAndErrorStruct;
             }
-            topicPartitionsStruct.set(PARTITIONS_KEY_NAME, partitionAndErrorsArray);
+            topicPartitionsStruct.set(PARTITIONS, partitionAndErrorsArray);
             partitionsArray[i++] = topicPartitionsStruct;
         }
 
-        struct.set(TOPICS_KEY_NAME, partitionsArray);
+        struct.set(TOPICS, partitionsArray);
         return struct;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index a273765704d4d..5c7b6f7eb7342 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -284,7 +284,7 @@ public String toString() {
 
     private UpdateMetadataRequest(short version, int controllerId, int controllerEpoch,
                                   Map partitionStates, Set liveBrokers) {
-        super(version);
+        super(ApiKeys.UPDATE_METADATA, version);
         this.controllerId = controllerId;
         this.controllerEpoch = controllerEpoch;
         this.partitionStates = partitionStates;
@@ -292,7 +292,7 @@ private UpdateMetadataRequest(short version, int controllerId, int controllerEpo
     }
 
     public UpdateMetadataRequest(Struct struct, short versionId) {
-        super(versionId);
+        super(ApiKeys.UPDATE_METADATA, versionId);
         Map partitionStates = new HashMap<>();
         for (Object partitionStateDataObj : struct.getArray(PARTITION_STATES_KEY_NAME)) {
             Struct partitionStateData = (Struct) partitionStateDataObj;
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java
index 3f7a0c9fc9b52..33f9bb586d9a5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java
@@ -153,13 +153,13 @@ public WriteTxnMarkersRequest build(short version) {
     private final List markers;
 
     private WriteTxnMarkersRequest(short version, List markers) {
-        super(version);
+        super(ApiKeys.WRITE_TXN_MARKERS, version);
 
         this.markers = markers;
     }
 
     public WriteTxnMarkersRequest(Struct struct, short version) {
-        super(version);
+        super(ApiKeys.WRITE_TXN_MARKERS, version);
         List markers = new ArrayList<>();
         Object[] markersArray = struct.getArray(TXN_MARKERS_KEY_NAME);
         for (Object markerObj : markersArray) {
@@ -204,7 +204,7 @@ protected Struct toStruct() {
             markerStruct.set(COORDINATOR_EPOCH_KEY_NAME, entry.coordinatorEpoch);
             markerStruct.set(TRANSACTION_RESULT_KEY_NAME, entry.result.id);
 
-            Map> mappedPartitions = CollectionUtils.groupDataByTopic(entry.partitions);
+            Map> mappedPartitions = CollectionUtils.groupPartitionsByTopic(entry.partitions);
             Object[] partitionsArray = new Object[mappedPartitions.size()];
             int j = 0;
             for (Map.Entry> topicAndPartitions : mappedPartitions.entrySet()) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java
index f307760e123f8..92b5fc0b29dc2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java
@@ -118,7 +118,7 @@ protected Struct toStruct(short version) {
             responseStruct.set(PRODUCER_ID_KEY_NAME, responseEntry.getKey());
 
             Map partitionAndErrors = responseEntry.getValue();
-            Map> mappedPartitions = CollectionUtils.groupDataByTopic(partitionAndErrors);
+            Map> mappedPartitions = CollectionUtils.groupPartitionDataByTopic(partitionAndErrors);
             Object[] partitionsArray = new Object[mappedPartitions.size()];
             int i = 0;
             for (Map.Entry> topicAndPartitions : mappedPartitions.entrySet()) {
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java
index 04fce647063e2..3489728c3f8e0 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java
@@ -39,41 +39,36 @@ public static  Map subtractMap(Map minuend
 
     /**
      * group data by topic
+     *
      * @param data Data to be partitioned
      * @param  Partition data type
      * @return partitioned data
      */
-    public static  Map> groupDataByTopic(Map data) {
+    public static  Map> groupPartitionDataByTopic(Map data) {
         Map> dataByTopic = new HashMap<>();
-        for (Map.Entry entry: data.entrySet()) {
+        for (Map.Entry entry : data.entrySet()) {
             String topic = entry.getKey().topic();
             int partition = entry.getKey().partition();
-            Map topicData = dataByTopic.get(topic);
-            if (topicData == null) {
-                topicData = new HashMap<>();
-                dataByTopic.put(topic, topicData);
-            }
+            Map topicData = dataByTopic.computeIfAbsent(topic, t -> new HashMap<>());
             topicData.put(partition, entry.getValue());
         }
         return dataByTopic;
     }
 
     /**
-     * group partitions by topic
-     * @param partitions
+     * Group a list of partitions by the topic name.
+     *
+     * @param partitions The partitions to collect
      * @return partitions per topic
      */
-    public static Map> groupDataByTopic(List partitions) {
+    public static Map> groupPartitionsByTopic(List partitions) {
         Map> partitionsByTopic = new HashMap<>();
-        for (TopicPartition tp: partitions) {
+        for (TopicPartition tp : partitions) {
             String topic = tp.topic();
-            List topicData = partitionsByTopic.get(topic);
-            if (topicData == null) {
-                topicData = new ArrayList<>();
-                partitionsByTopic.put(topic, topicData);
-            }
+            List topicData = partitionsByTopic.computeIfAbsent(topic, t -> new ArrayList<>());
             topicData.add(tp.partition());
         }
-        return  partitionsByTopic;
+        return partitionsByTopic;
     }
+
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java
index 3095717e8dd72..4c12fc68babcf 100644
--- a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java
@@ -18,13 +18,13 @@
 
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.MemoryRecords;
 import org.apache.kafka.common.requests.FetchRequest;
 import org.apache.kafka.common.requests.FetchResponse;
 import org.apache.kafka.common.utils.LogContext;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
-import org.slf4j.Logger;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -33,6 +33,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -52,13 +53,11 @@ public class FetchSessionHandlerTest {
 
     private static final LogContext LOG_CONTEXT = new LogContext("[FetchSessionHandler]=");
 
-    private static final Logger log = LOG_CONTEXT.logger(FetchSessionHandler.class);
-
     /**
      * Create a set of TopicPartitions.  We use a TreeSet, in order to get a deterministic
      * ordering for test purposes.
      */
-    private final static Set toSet(TopicPartition... arr) {
+    private static Set toSet(TopicPartition... arr) {
         TreeSet set = new TreeSet<>(new Comparator() {
             @Override
             public int compare(TopicPartition o1, TopicPartition o2) {
@@ -70,7 +69,7 @@ public int compare(TopicPartition o1, TopicPartition o2) {
     }
 
     @Test
-    public void testFindMissing() throws Exception {
+    public void testFindMissing() {
         TopicPartition foo0 = new TopicPartition("foo", 0);
         TopicPartition foo1 = new TopicPartition("foo", 1);
         TopicPartition bar0 = new TopicPartition("bar", 0);
@@ -95,7 +94,7 @@ private static final class ReqEntry {
 
         ReqEntry(String topic, int partition, long fetchOffset, long logStartOffset, int maxBytes) {
             this.part = new TopicPartition(topic, partition);
-            this.data = new FetchRequest.PartitionData(fetchOffset, logStartOffset, maxBytes);
+            this.data = new FetchRequest.PartitionData(fetchOffset, logStartOffset, maxBytes, Optional.empty());
         }
     }
 
@@ -153,11 +152,11 @@ private static void assertListEquals(List expected, List data;
 
         RespEntry(String topic, int partition, long highWatermark, long lastStableOffset) {
             this.part = new TopicPartition(topic, partition);
-            this.data = new FetchResponse.PartitionData(
+            this.data = new FetchResponse.PartitionData<>(
                 Errors.NONE,
                 highWatermark,
                 lastStableOffset,
@@ -167,8 +166,8 @@ private static final class RespEntry {
         }
     }
 
-    private static LinkedHashMap respMap(RespEntry... entries) {
-        LinkedHashMap map = new LinkedHashMap<>();
+    private static LinkedHashMap> respMap(RespEntry... entries) {
+        LinkedHashMap> map = new LinkedHashMap<>();
         for (RespEntry entry : entries) {
             map.put(entry.part, entry.data);
         }
@@ -180,13 +179,13 @@ private static LinkedHashMap respMa
      * Pre-KIP-227 brokers always supply this kind of response.
      */
     @Test
-    public void testSessionless() throws Exception {
+    public void testSessionless() {
         FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1);
         FetchSessionHandler.Builder builder = handler.newBuilder();
         builder.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         builder.add(new TopicPartition("foo", 1),
-            new FetchRequest.PartitionData(10, 110, 210));
+            new FetchRequest.PartitionData(10, 110, 210, Optional.empty()));
         FetchSessionHandler.FetchRequestData data = builder.build();
         assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200),
                                new ReqEntry("foo", 1, 10, 110, 210)),
@@ -194,7 +193,7 @@ public void testSessionless() throws Exception {
         assertEquals(INVALID_SESSION_ID, data.metadata().sessionId());
         assertEquals(INITIAL_EPOCH, data.metadata().epoch());
 
-        FetchResponse resp = new FetchResponse(Errors.NONE,
+        FetchResponse resp = new FetchResponse<>(Errors.NONE,
             respMap(new RespEntry("foo", 0, 0, 0),
                     new RespEntry("foo", 1, 0, 0)),
             0, INVALID_SESSION_ID);
@@ -202,7 +201,7 @@ public void testSessionless() throws Exception {
 
         FetchSessionHandler.Builder builder2 = handler.newBuilder();
         builder2.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         FetchSessionHandler.FetchRequestData data2 = builder2.build();
         assertEquals(INVALID_SESSION_ID, data2.metadata().sessionId());
         assertEquals(INITIAL_EPOCH, data2.metadata().epoch());
@@ -214,13 +213,13 @@ public void testSessionless() throws Exception {
      * Test handling an incremental fetch session.
      */
     @Test
-    public void testIncrementals() throws Exception {
+    public void testIncrementals() {
         FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1);
         FetchSessionHandler.Builder builder = handler.newBuilder();
         builder.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         builder.add(new TopicPartition("foo", 1),
-            new FetchRequest.PartitionData(10, 110, 210));
+            new FetchRequest.PartitionData(10, 110, 210, Optional.empty()));
         FetchSessionHandler.FetchRequestData data = builder.build();
         assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200),
             new ReqEntry("foo", 1, 10, 110, 210)),
@@ -228,7 +227,7 @@ public void testIncrementals() throws Exception {
         assertEquals(INVALID_SESSION_ID, data.metadata().sessionId());
         assertEquals(INITIAL_EPOCH, data.metadata().epoch());
 
-        FetchResponse resp = new FetchResponse(Errors.NONE,
+        FetchResponse resp = new FetchResponse<>(Errors.NONE,
             respMap(new RespEntry("foo", 0, 10, 20),
                     new RespEntry("foo", 1, 10, 20)),
             0, 123);
@@ -237,11 +236,11 @@ public void testIncrementals() throws Exception {
         // Test an incremental fetch request which adds one partition and modifies another.
         FetchSessionHandler.Builder builder2 = handler.newBuilder();
         builder2.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         builder2.add(new TopicPartition("foo", 1),
-            new FetchRequest.PartitionData(10, 120, 210));
+            new FetchRequest.PartitionData(10, 120, 210, Optional.empty()));
         builder2.add(new TopicPartition("bar", 0),
-            new FetchRequest.PartitionData(20, 200, 200));
+            new FetchRequest.PartitionData(20, 200, 200, Optional.empty()));
         FetchSessionHandler.FetchRequestData data2 = builder2.build();
         assertFalse(data2.metadata().isFull());
         assertMapEquals(reqMap(new ReqEntry("foo", 0, 0, 100, 200),
@@ -252,24 +251,24 @@ public void testIncrementals() throws Exception {
                 new ReqEntry("foo", 1, 10, 120, 210)),
             data2.toSend());
 
-        FetchResponse resp2 = new FetchResponse(Errors.NONE,
+        FetchResponse resp2 = new FetchResponse<>(Errors.NONE,
             respMap(new RespEntry("foo", 1, 20, 20)),
             0, 123);
         handler.handleResponse(resp2);
 
         // Skip building a new request.  Test that handling an invalid fetch session epoch response results
         // in a request which closes the session.
-        FetchResponse resp3 = new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, respMap(),
+        FetchResponse resp3 = new FetchResponse<>(Errors.INVALID_FETCH_SESSION_EPOCH, respMap(),
             0, INVALID_SESSION_ID);
         handler.handleResponse(resp3);
 
         FetchSessionHandler.Builder builder4 = handler.newBuilder();
         builder4.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         builder4.add(new TopicPartition("foo", 1),
-            new FetchRequest.PartitionData(10, 120, 210));
+            new FetchRequest.PartitionData(10, 120, 210, Optional.empty()));
         builder4.add(new TopicPartition("bar", 0),
-            new FetchRequest.PartitionData(20, 200, 200));
+            new FetchRequest.PartitionData(20, 200, 200, Optional.empty()));
         FetchSessionHandler.FetchRequestData data4 = builder4.build();
         assertTrue(data4.metadata().isFull());
         assertEquals(data2.metadata().sessionId(), data4.metadata().sessionId());
@@ -284,11 +283,11 @@ public void testIncrementals() throws Exception {
      * Test that calling FetchSessionHandler#Builder#build twice fails.
      */
     @Test
-    public void testDoubleBuild() throws Exception {
+    public void testDoubleBuild() {
         FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1);
         FetchSessionHandler.Builder builder = handler.newBuilder();
         builder.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         builder.build();
         try {
             builder.build();
@@ -299,15 +298,15 @@ public void testDoubleBuild() throws Exception {
     }
 
     @Test
-    public void testIncrementalPartitionRemoval() throws Exception {
+    public void testIncrementalPartitionRemoval() {
         FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1);
         FetchSessionHandler.Builder builder = handler.newBuilder();
         builder.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         builder.add(new TopicPartition("foo", 1),
-            new FetchRequest.PartitionData(10, 110, 210));
+            new FetchRequest.PartitionData(10, 110, 210, Optional.empty()));
         builder.add(new TopicPartition("bar", 0),
-            new FetchRequest.PartitionData(20, 120, 220));
+            new FetchRequest.PartitionData(20, 120, 220, Optional.empty()));
         FetchSessionHandler.FetchRequestData data = builder.build();
         assertMapsEqual(reqMap(new ReqEntry("foo", 0, 0, 100, 200),
             new ReqEntry("foo", 1, 10, 110, 210),
@@ -315,7 +314,7 @@ public void testIncrementalPartitionRemoval() throws Exception {
             data.toSend(), data.sessionPartitions());
         assertTrue(data.metadata().isFull());
 
-        FetchResponse resp = new FetchResponse(Errors.NONE,
+        FetchResponse resp = new FetchResponse<>(Errors.NONE,
             respMap(new RespEntry("foo", 0, 10, 20),
                     new RespEntry("foo", 1, 10, 20),
                     new RespEntry("bar", 0, 10, 20)),
@@ -325,7 +324,7 @@ public void testIncrementalPartitionRemoval() throws Exception {
         // Test an incremental fetch request which removes two partitions.
         FetchSessionHandler.Builder builder2 = handler.newBuilder();
         builder2.add(new TopicPartition("foo", 1),
-            new FetchRequest.PartitionData(10, 110, 210));
+            new FetchRequest.PartitionData(10, 110, 210, Optional.empty()));
         FetchSessionHandler.FetchRequestData data2 = builder2.build();
         assertFalse(data2.metadata().isFull());
         assertEquals(123, data2.metadata().sessionId());
@@ -340,12 +339,12 @@ public void testIncrementalPartitionRemoval() throws Exception {
 
         // A FETCH_SESSION_ID_NOT_FOUND response triggers us to close the session.
         // The next request is a session establishing FULL request.
-        FetchResponse resp2 = new FetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND,
+        FetchResponse resp2 = new FetchResponse<>(Errors.FETCH_SESSION_ID_NOT_FOUND,
             respMap(), 0, INVALID_SESSION_ID);
         handler.handleResponse(resp2);
         FetchSessionHandler.Builder builder3 = handler.newBuilder();
         builder3.add(new TopicPartition("foo", 0),
-            new FetchRequest.PartitionData(0, 100, 200));
+            new FetchRequest.PartitionData(0, 100, 200, Optional.empty()));
         FetchSessionHandler.FetchRequestData data3 = builder3.build();
         assertTrue(data3.metadata().isFull());
         assertEquals(INVALID_SESSION_ID, data3.metadata().sessionId());
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index c0dc542b15954..34432c31034a4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -94,6 +94,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -462,12 +463,13 @@ public void testMetadataRetries() throws Exception {
             env.kafkaClient().prepareResponse(new MetadataResponse(initializedCluster.nodes(),
                     initializedCluster.clusterResource().clusterId(),
                     initializedCluster.controller().id(),
-                    Collections.emptyList()));
+                    Collections.emptyList()));
 
             // Then we respond to the DescribeTopic request
             Node leader = initializedCluster.nodes().get(0);
             MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata(
-                    Errors.NONE, 0, leader, singletonList(leader), singletonList(leader), singletonList(leader));
+                    Errors.NONE, 0, leader, Optional.of(10), singletonList(leader),
+                    singletonList(leader), singletonList(leader));
             env.kafkaClient().prepareResponse(new MetadataResponse(initializedCluster.nodes(),
                     initializedCluster.clusterResource().clusterId(), 1,
                     singletonList(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false,
@@ -788,16 +790,17 @@ public void testDeleteRecords() throws Exception {
 
             List t = new ArrayList<>();
             List p = new ArrayList<>();
-            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 0, nodes.get(0),
-                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
-            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, nodes.get(0),
-                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
+            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 0, nodes.get(0), Optional.of(5),
+                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
+            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, nodes.get(0), Optional.of(5),
+                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
             p.add(new MetadataResponse.PartitionMetadata(Errors.LEADER_NOT_AVAILABLE, 2, null,
-                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
-            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 3, nodes.get(0),
-                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
-            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 4, nodes.get(0),
-                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
+                    Optional.empty(), singletonList(nodes.get(0)), singletonList(nodes.get(0)),
+                    Collections.emptyList()));
+            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 3, nodes.get(0), Optional.of(5),
+                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
+            p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 4, nodes.get(0), Optional.of(5),
+                    singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
 
             t.add(new MetadataResponse.TopicMetadata(Errors.NONE, "my_topic", false, p));
 
@@ -1068,9 +1071,9 @@ public void testDescribeConsumerGroupOffsets() throws Exception {
             new Cluster(
                 "mockClusterId",
                 nodes.values(),
-                Collections.emptyList(),
-                Collections.emptySet(),
-                Collections.emptySet(), nodes.get(0));
+                Collections.emptyList(),
+                Collections.emptySet(),
+                Collections.emptySet(), nodes.get(0));
 
         try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
             env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
@@ -1083,9 +1086,12 @@ public void testDescribeConsumerGroupOffsets() throws Exception {
             TopicPartition myTopicPartition2 = new TopicPartition("my_topic", 2);
 
             final Map responseData = new HashMap<>();
-            responseData.put(myTopicPartition0, new OffsetFetchResponse.PartitionData(10, "", Errors.NONE));
-            responseData.put(myTopicPartition1, new OffsetFetchResponse.PartitionData(0, "", Errors.NONE));
-            responseData.put(myTopicPartition2, new OffsetFetchResponse.PartitionData(20, "", Errors.NONE));
+            responseData.put(myTopicPartition0, new OffsetFetchResponse.PartitionData(10,
+                    Optional.empty(), "", Errors.NONE));
+            responseData.put(myTopicPartition1, new OffsetFetchResponse.PartitionData(0,
+                    Optional.empty(), "", Errors.NONE));
+            responseData.put(myTopicPartition2, new OffsetFetchResponse.PartitionData(20,
+                    Optional.empty(), "", Errors.NONE));
             env.kafkaClient().prepareResponse(new OffsetFetchResponse(Errors.NONE, responseData));
 
             final ListConsumerGroupOffsetsResult result = env.adminClient().listConsumerGroupOffsets("group-0");
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerRecordTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerRecordTest.java
index 327364520040c..aae269adf2bde 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerRecordTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerRecordTest.java
@@ -21,6 +21,8 @@
 import org.apache.kafka.common.record.TimestampType;
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.junit.Assert.assertEquals;
 
 public class ConsumerRecordTest {
@@ -45,6 +47,7 @@ public void testOldConstructor() {
         assertEquals(ConsumerRecord.NULL_CHECKSUM, record.checksum());
         assertEquals(ConsumerRecord.NULL_SIZE, record.serializedKeySize());
         assertEquals(ConsumerRecord.NULL_SIZE, record.serializedValueSize());
+        assertEquals(Optional.empty(), record.leaderEpoch());
         assertEquals(new RecordHeaders(), record.headers());
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index a0f95c4aabf8b..8b8cea6cce1b7 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.clients.consumer;
 
-import java.util.ArrayList;
 import org.apache.kafka.clients.ClientRequest;
 import org.apache.kafka.clients.KafkaClient;
 import org.apache.kafka.clients.Metadata;
@@ -83,6 +82,7 @@
 
 import java.nio.ByteBuffer;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -91,6 +91,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Queue;
 import java.util.Set;
@@ -499,7 +500,7 @@ public void testFetchProgressWithMissingPartitionPosition() {
         Node node = cluster.nodes().get(0);
 
         Metadata metadata = createMetadata();
-        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
+        metadata.update(cluster, Collections.emptySet(), time.milliseconds());
 
         MockClient client = new MockClient(time, metadata);
         client.setNode(node);
@@ -514,10 +515,9 @@ public void testFetchProgressWithMissingPartitionPosition() {
                     @Override
                     public boolean matches(AbstractRequest body) {
                         ListOffsetRequest request = (ListOffsetRequest) body;
-                        Map expectedTimestamps = new HashMap<>();
-                        expectedTimestamps.put(tp0, ListOffsetRequest.LATEST_TIMESTAMP);
-                        expectedTimestamps.put(tp1, ListOffsetRequest.EARLIEST_TIMESTAMP);
-                        return expectedTimestamps.equals(request.partitionTimestamps());
+                        Map timestamps = request.partitionTimestamps();
+                        return timestamps.get(tp0).timestamp == ListOffsetRequest.LATEST_TIMESTAMP &&
+                                timestamps.get(tp1).timestamp == ListOffsetRequest.EARLIEST_TIMESTAMP;
                     }
                 }, listOffsetsResponse(Collections.singletonMap(tp0, 50L),
                         Collections.singletonMap(tp1, Errors.NOT_LEADER_FOR_PARTITION)));
@@ -1644,7 +1644,7 @@ private OffsetCommitResponse offsetCommitResponse(Map re
 
     private JoinGroupResponse joinGroupFollowerResponse(PartitionAssignor assignor, int generationId, String memberId, String leaderId, Errors error) {
         return new JoinGroupResponse(error, generationId, assignor.name(), memberId, leaderId,
-                Collections.emptyMap());
+                Collections.emptyMap());
     }
 
     private SyncGroupResponse syncGroupResponse(List partitions, Errors error) {
@@ -1655,7 +1655,8 @@ private SyncGroupResponse syncGroupResponse(List partitions, Err
     private OffsetFetchResponse offsetResponse(Map offsets, Errors error) {
         Map partitionData = new HashMap<>();
         for (Map.Entry entry : offsets.entrySet()) {
-            partitionData.put(entry.getKey(), new OffsetFetchResponse.PartitionData(entry.getValue(), "", error));
+            partitionData.put(entry.getKey(), new OffsetFetchResponse.PartitionData(entry.getValue(),
+                    Optional.empty(), "", error));
         }
         return new OffsetFetchResponse(Errors.NONE, partitionData);
     }
@@ -1669,13 +1670,14 @@ private ListOffsetResponse listOffsetsResponse(Map partiti
         Map partitionData = new HashMap<>();
         for (Map.Entry partitionOffset : partitionOffsets.entrySet()) {
             partitionData.put(partitionOffset.getKey(), new ListOffsetResponse.PartitionData(Errors.NONE,
-                    ListOffsetResponse.UNKNOWN_TIMESTAMP, partitionOffset.getValue()));
+                    ListOffsetResponse.UNKNOWN_TIMESTAMP, partitionOffset.getValue(),
+                    Optional.empty()));
         }
 
         for (Map.Entry partitionError : partitionErrors.entrySet()) {
             partitionData.put(partitionError.getKey(), new ListOffsetResponse.PartitionData(
                     partitionError.getValue(), ListOffsetResponse.UNKNOWN_TIMESTAMP,
-                    ListOffsetResponse.UNKNOWN_OFFSET));
+                    ListOffsetResponse.UNKNOWN_OFFSET, Optional.empty()));
         }
 
         return new ListOffsetResponse(partitionData);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/SerializeCompatibilityOffsetAndMetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/OffsetAndMetadataTest.java
similarity index 50%
rename from clients/src/test/java/org/apache/kafka/clients/consumer/SerializeCompatibilityOffsetAndMetadataTest.java
rename to clients/src/test/java/org/apache/kafka/clients/consumer/OffsetAndMetadataTest.java
index 324aeafd88646..5bdbf7c7a3279 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/SerializeCompatibilityOffsetAndMetadataTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/OffsetAndMetadataTest.java
@@ -20,9 +20,8 @@
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.Optional;
 
-
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -30,35 +29,38 @@
  * Note: this ensures that the current code can deserialize data serialized with older versions of the code, but not the reverse.
  * That is, older code won't necessarily be able to deserialize data serialized with newer code.
  */
-public class SerializeCompatibilityOffsetAndMetadataTest {
-    private String metadata = "test commit metadata";
-    private String fileName = "serializedData/offsetAndMetadataSerializedfile";
-    private long offset = 10;
+public class OffsetAndMetadataTest {
 
-    private void checkValues(OffsetAndMetadata deSerOAM) {
-        //assert deserialized values are same as original
-        assertEquals("Offset should be " + offset + " but got " + deSerOAM.offset(), offset, deSerOAM.offset());
-        assertEquals("metadata should be " + metadata + " but got " + deSerOAM.metadata(), metadata, deSerOAM.metadata());
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidNegativeOffset() {
+        new OffsetAndMetadata(-239L, Optional.of(15), "");
     }
 
     @Test
     public void testSerializationRoundtrip() throws IOException, ClassNotFoundException {
-        //assert OffsetAndMetadata is serializable
-        OffsetAndMetadata origOAM = new OffsetAndMetadata(offset, metadata);
-        byte[] byteArray =  Serializer.serialize(origOAM);
+        checkSerde(new OffsetAndMetadata(239L, Optional.of(15), "blah"));
+        checkSerde(new OffsetAndMetadata(239L, "blah"));
+        checkSerde(new OffsetAndMetadata(239L));
+    }
 
-        //deserialize the byteArray and check if the values are same as original
-        Object deserializedObject = Serializer.deserialize(byteArray);
-        assertTrue(deserializedObject instanceof OffsetAndMetadata);
-        checkValues((OffsetAndMetadata) deserializedObject);
+    private void checkSerde(OffsetAndMetadata offsetAndMetadata) throws IOException, ClassNotFoundException {
+        byte[] bytes =  Serializer.serialize(offsetAndMetadata);
+        OffsetAndMetadata deserialized = (OffsetAndMetadata) Serializer.deserialize(bytes);
+        assertEquals(offsetAndMetadata, deserialized);
     }
 
     @Test
-    public void testOffsetMetadataSerializationCompatibility() throws IOException, ClassNotFoundException {
-        // assert serialized OffsetAndMetadata object in file (oamserializedfile under resources folder) is
-        // deserializable into OffsetAndMetadata and is compatible
+    public void testDeserializationCompatibilityBeforeLeaderEpoch() throws IOException, ClassNotFoundException {
+        String fileName = "serializedData/offsetAndMetadataBeforeLeaderEpoch";
         Object deserializedObject = Serializer.deserialize(fileName);
-        assertTrue(deserializedObject instanceof OffsetAndMetadata);
-        checkValues((OffsetAndMetadata) deserializedObject);
+        assertEquals(new OffsetAndMetadata(10, "test commit metadata"), deserializedObject);
     }
+
+    @Test
+    public void testDeserializationCompatibilityWithLeaderEpoch() throws IOException, ClassNotFoundException {
+        String fileName = "serializedData/offsetAndMetadataWithLeaderEpoch";
+        Object deserializedObject = Serializer.deserialize(fileName);
+        assertEquals(new OffsetAndMetadata(10, Optional.of(235), "test commit metadata"), deserializedObject);
+    }
+
 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
index 90e3b99870c76..31cee7edbf03d 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/StickyAssignorTest.java
@@ -711,8 +711,8 @@ private static void verifyValidityAndBalance(Map subscript
                 if (Math.abs(len - otherLen) <= 1)
                     continue;
 
-                Map> map = CollectionUtils.groupDataByTopic(partitions);
-                Map> otherMap = CollectionUtils.groupDataByTopic(otherPartitions);
+                Map> map = CollectionUtils.groupPartitionsByTopic(partitions);
+                Map> otherMap = CollectionUtils.groupPartitionsByTopic(otherPartitions);
 
                 if (len > otherLen) {
                     for (String topic: map.keySet())
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 2b6d30377286f..af073f1d56030 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -42,6 +42,7 @@
 import org.apache.kafka.common.errors.WakeupException;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatResponse;
@@ -72,6 +73,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -88,6 +90,7 @@
 import static java.util.Collections.singletonMap;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -245,7 +248,8 @@ public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() throws
 
         final AtomicBoolean asyncCallbackInvoked = new AtomicBoolean(false);
         Map offsets = singletonMap(
-                new TopicPartition("foo", 0), new OffsetCommitRequest.PartitionData(13L, ""));
+                new TopicPartition("foo", 0), new OffsetCommitRequest.PartitionData(13L,
+                        RecordBatch.NO_PARTITION_LEADER_EPOCH, ""));
         consumerClient.send(coordinator.checkAndGetCoordinator(), new OffsetCommitRequest.Builder(groupId, offsets))
                 .compose(new RequestFutureAdapter() {
                     @Override
@@ -1571,22 +1575,6 @@ public void testCommitOffsetSyncCallbackWithNonRetriableException() {
         coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(100L)), time.timer(Long.MAX_VALUE));
     }
 
-    @Test(expected = IllegalArgumentException.class)
-    public void testCommitSyncNegativeOffset() {
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.commitOffsetsSync(singletonMap(t1p, new OffsetAndMetadata(-1L)), time.timer(Long.MAX_VALUE));
-    }
-
-    @Test
-    public void testCommitAsyncNegativeOffset() {
-        int invokedBeforeTest = mockOffsetCommitCallback.invoked;
-        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
-        coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(-1L)), mockOffsetCommitCallback);
-        coordinator.invokeCompletedOffsetCommitCallbacks();
-        assertEquals(invokedBeforeTest + 1, mockOffsetCommitCallback.invoked);
-        assertTrue(mockOffsetCommitCallback.exception instanceof IllegalArgumentException);
-    }
-
     @Test
     public void testCommitOffsetSyncWithoutFutureGetsCompleted() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
@@ -1608,6 +1596,25 @@ public void testRefreshOffset() {
         assertEquals(100L, subscriptions.position(t1p).longValue());
     }
 
+    @Test
+    public void testFetchCommittedOffsets() {
+        client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+        coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
+
+        long offset = 500L;
+        String metadata = "blahblah";
+        Optional leaderEpoch = Optional.of(15);
+        OffsetFetchResponse.PartitionData data = new OffsetFetchResponse.PartitionData(offset, leaderEpoch,
+                metadata, Errors.NONE);
+
+        client.prepareResponse(new OffsetFetchResponse(Errors.NONE, singletonMap(t1p, data)));
+        Map fetchedOffsets = coordinator.fetchCommittedOffsets(singleton(t1p),
+                time.timer(Long.MAX_VALUE));
+
+        assertNotNull(fetchedOffsets);
+        assertEquals(new OffsetAndMetadata(offset, leaderEpoch, metadata), fetchedOffsets.get(t1p));
+    }
+
     @Test
     public void testRefreshOffsetLoadInProgress() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
@@ -2074,7 +2081,8 @@ private OffsetFetchResponse offsetFetchResponse(Errors topLevelError) {
     }
 
     private OffsetFetchResponse offsetFetchResponse(TopicPartition tp, Errors partitionLevelError, String metadata, long offset) {
-        OffsetFetchResponse.PartitionData data = new OffsetFetchResponse.PartitionData(offset, metadata, partitionLevelError);
+        OffsetFetchResponse.PartitionData data = new OffsetFetchResponse.PartitionData(offset,
+                Optional.empty(), metadata, partitionLevelError);
         return new OffsetFetchResponse(Errors.NONE, singletonMap(tp, data));
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index fd550d61da515..d314a4d8c9feb 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -62,11 +62,10 @@
 import org.apache.kafka.common.record.Records;
 import org.apache.kafka.common.record.SimpleRecord;
 import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.requests.FetchResponse;
 import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.ApiVersionsResponse;
 import org.apache.kafka.common.requests.FetchRequest;
-import org.apache.kafka.common.requests.FetchRequest.PartitionData;
+import org.apache.kafka.common.requests.FetchResponse;
 import org.apache.kafka.common.requests.IsolationLevel;
 import org.apache.kafka.common.requests.ListOffsetRequest;
 import org.apache.kafka.common.requests.ListOffsetResponse;
@@ -100,6 +99,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static java.util.Collections.singleton;
@@ -208,6 +208,85 @@ public void testFetchNormal() {
         }
     }
 
+    @Test
+    public void testMissingLeaderEpochInRecords() {
+        subscriptions.assignFromUser(singleton(tp0));
+        subscriptions.seek(tp0, 0);
+
+        ByteBuffer buffer = ByteBuffer.allocate(1024);
+        MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0,
+                CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(),
+                RecordBatch.NO_PARTITION_LEADER_EPOCH);
+        builder.append(0L, "key".getBytes(), "1".getBytes());
+        builder.append(0L, "key".getBytes(), "2".getBytes());
+        MemoryRecords records = builder.build();
+
+        assertEquals(1, fetcher.sendFetches());
+        assertFalse(fetcher.hasCompletedFetches());
+
+        client.prepareResponse(fullFetchResponse(tp0, records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+        assertTrue(fetcher.hasCompletedFetches());
+
+        Map>> partitionRecords = fetcher.fetchedRecords();
+        assertTrue(partitionRecords.containsKey(tp0));
+        assertEquals(2, partitionRecords.get(tp0).size());
+
+        for (ConsumerRecord record : partitionRecords.get(tp0)) {
+            assertEquals(Optional.empty(), record.leaderEpoch());
+        }
+    }
+
+    @Test
+    public void testLeaderEpochInConsumerRecord() {
+        subscriptions.assignFromUser(singleton(tp0));
+        subscriptions.seek(tp0, 0);
+
+        Integer partitionLeaderEpoch = 1;
+
+        ByteBuffer buffer = ByteBuffer.allocate(1024);
+        MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE,
+                CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(),
+                partitionLeaderEpoch);
+        builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
+        builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
+        builder.close();
+
+        partitionLeaderEpoch += 7;
+
+        builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE,
+                TimestampType.CREATE_TIME, 2L, System.currentTimeMillis(), partitionLeaderEpoch);
+        builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
+        builder.close();
+
+        partitionLeaderEpoch += 5;
+        builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE,
+                TimestampType.CREATE_TIME, 3L, System.currentTimeMillis(), partitionLeaderEpoch);
+        builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
+        builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
+        builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes());
+        builder.close();
+
+        buffer.flip();
+        MemoryRecords records = MemoryRecords.readableRecords(buffer);
+
+        assertEquals(1, fetcher.sendFetches());
+        assertFalse(fetcher.hasCompletedFetches());
+
+        client.prepareResponse(fullFetchResponse(tp0, records, Errors.NONE, 100L, 0));
+        consumerClient.poll(time.timer(0));
+        assertTrue(fetcher.hasCompletedFetches());
+
+        Map>> partitionRecords = fetcher.fetchedRecords();
+        assertTrue(partitionRecords.containsKey(tp0));
+        assertEquals(6, partitionRecords.get(tp0).size());
+
+        for (ConsumerRecord record : partitionRecords.get(tp0)) {
+            int expectedLeaderEpoch = Integer.parseInt(Utils.utf8(record.value()));
+            assertEquals(Optional.of(expectedLeaderEpoch), record.leaderEpoch());
+        }
+    }
+
     @Test
     public void testFetchSkipsBlackedOutNodes() {
         subscriptions.assignFromUser(singleton(tp0));
@@ -1403,6 +1482,7 @@ public void testGetTopicMetadataOfflinePartitions() {
                     p.error(),
                     p.partition(),
                     null, //no leader
+                    Optional.empty(),
                     p.replicas(),
                     p.isr(),
                     p.offlineReplicas())
@@ -1843,9 +1923,11 @@ public void testGetOffsetsForTimes() {
     public void testBatchedListOffsetsMetadataErrors() {
         Map partitionData = new HashMap<>();
         partitionData.put(tp0, new ListOffsetResponse.PartitionData(Errors.NOT_LEADER_FOR_PARTITION,
-                ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET));
+                ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET,
+                Optional.empty()));
         partitionData.put(tp1, new ListOffsetResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION,
-                ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET));
+                ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET,
+                Optional.empty()));
         client.prepareResponse(new ListOffsetResponse(0, partitionData));
 
         Map offsetsToSearch = new HashMap<>();
@@ -2454,7 +2536,8 @@ private void testGetOffsetsForTimesWithUnknownOffset() {
 
         Map partitionData = new HashMap<>();
         partitionData.put(tp0, new ListOffsetResponse.PartitionData(Errors.NONE,
-                ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET));
+                ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET,
+                Optional.empty()));
 
         client.prepareResponseFrom(new ListOffsetResponse(0, partitionData), cluster.leaderFor(tp0));
 
@@ -2473,7 +2556,7 @@ private MockClient.RequestMatcher listOffsetRequestMatcher(final long timestamp)
             @Override
             public boolean matches(AbstractRequest body) {
                 ListOffsetRequest req = (ListOffsetRequest) body;
-                return timestamp == req.partitionTimestamps().get(tp0);
+                return timestamp == req.partitionTimestamps().get(tp0).timestamp;
             }
         };
     }
@@ -2483,7 +2566,8 @@ private ListOffsetResponse listOffsetResponse(Errors error, long timestamp, long
     }
 
     private ListOffsetResponse listOffsetResponse(TopicPartition tp, Errors error, long timestamp, long offset) {
-        ListOffsetResponse.PartitionData partitionData = new ListOffsetResponse.PartitionData(error, timestamp, offset);
+        ListOffsetResponse.PartitionData partitionData = new ListOffsetResponse.PartitionData(error, timestamp, offset,
+                Optional.empty());
         Map allPartitionData = new HashMap<>();
         allPartitionData.put(tp, partitionData);
         return new ListOffsetResponse(allPartitionData);
@@ -2526,6 +2610,7 @@ private MetadataResponse newMetadataResponse(String topic, Errors error) {
                         Errors.NONE,
                         partitionInfo.partition(),
                         partitionInfo.leader(),
+                        Optional.empty(),
                         Arrays.asList(partitionInfo.replicas()),
                         Arrays.asList(partitionInfo.inSyncReplicas()),
                         Arrays.asList(partitionInfo.offlineReplicas())));
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
index 606fa714a3420..cf730b9cb120b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -2429,19 +2429,17 @@ public boolean matches(AbstractRequest body) {
         }, new AddOffsetsToTxnResponse(0, error));
     }
 
-    private void prepareTxnOffsetCommitResponse(final String consumerGroupId, final long producerId,
-                                                final short producerEpoch, Map txnOffsetCommitResponse) {
-        client.prepareResponse(new MockClient.RequestMatcher() {
-            @Override
-            public boolean matches(AbstractRequest body) {
-                TxnOffsetCommitRequest txnOffsetCommitRequest = (TxnOffsetCommitRequest) body;
-                assertEquals(consumerGroupId, txnOffsetCommitRequest.consumerGroupId());
-                assertEquals(producerId, txnOffsetCommitRequest.producerId());
-                assertEquals(producerEpoch, txnOffsetCommitRequest.producerEpoch());
-                return true;
-            }
+    private void prepareTxnOffsetCommitResponse(final String consumerGroupId,
+                                                final long producerId,
+                                                final short producerEpoch,
+                                                Map txnOffsetCommitResponse) {
+        client.prepareResponse(request -> {
+            TxnOffsetCommitRequest txnOffsetCommitRequest = (TxnOffsetCommitRequest) request;
+            assertEquals(consumerGroupId, txnOffsetCommitRequest.consumerGroupId());
+            assertEquals(producerId, txnOffsetCommitRequest.producerId());
+            assertEquals(producerEpoch, txnOffsetCommitRequest.producerEpoch());
+            return true;
         }, new TxnOffsetCommitResponse(0, txnOffsetCommitResponse));
-
     }
 
     private ProduceResponse produceResponse(TopicPartition tp, long offset, Errors error, int throttleTimeMs) {
diff --git a/clients/src/test/java/org/apache/kafka/common/SerializeCompatibilityTopicPartitionTest.java b/clients/src/test/java/org/apache/kafka/common/TopicPartitionTest.java
similarity index 98%
rename from clients/src/test/java/org/apache/kafka/common/SerializeCompatibilityTopicPartitionTest.java
rename to clients/src/test/java/org/apache/kafka/common/TopicPartitionTest.java
index 8b4df5ff01f6a..2a90338c64e6c 100644
--- a/clients/src/test/java/org/apache/kafka/common/SerializeCompatibilityTopicPartitionTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/TopicPartitionTest.java
@@ -29,8 +29,7 @@
  * Note: this ensures that the current code can deserialize data serialized with older versions of the code, but not the reverse.
  * That is, older code won't necessarily be able to deserialize data serialized with newer code.
  */
-public class SerializeCompatibilityTopicPartitionTest {
-
+public class TopicPartitionTest {
     private String topicName = "mytopic";
     private String fileName = "serializedData/topicPartitionSerializedfile";
     private int partNum = 5;
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index da613982816c7..05b9926d6cab9 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -72,6 +72,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static java.util.Arrays.asList;
@@ -469,14 +470,15 @@ public void produceResponseVersionTest() {
 
     @Test
     public void fetchResponseVersionTest() {
-        LinkedHashMap responseData = new LinkedHashMap<>();
+        LinkedHashMap> responseData = new LinkedHashMap<>();
 
         MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10));
-        responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE, 1000000,
-                FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records));
+        responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(
+                Errors.NONE, 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET,
+                0L, null, records));
 
-        FetchResponse v0Response = new FetchResponse(Errors.NONE, responseData, 0, INVALID_SESSION_ID);
-        FetchResponse v1Response = new FetchResponse(Errors.NONE, responseData, 10, INVALID_SESSION_ID);
+        FetchResponse v0Response = new FetchResponse<>(Errors.NONE, responseData, 0, INVALID_SESSION_ID);
+        FetchResponse v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID);
         assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs());
         assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs());
         assertEquals("Should use schema version 0", ApiKeys.FETCH.responseSchema((short) 0),
@@ -489,22 +491,21 @@ public void fetchResponseVersionTest() {
 
     @Test
     public void testFetchResponseV4() {
-        LinkedHashMap responseData = new LinkedHashMap<>();
-
+        LinkedHashMap> responseData = new LinkedHashMap<>();
         MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10));
 
         List abortedTransactions = asList(
                 new FetchResponse.AbortedTransaction(10, 100),
                 new FetchResponse.AbortedTransaction(15, 50)
         );
-        responseData.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData(Errors.NONE, 100000,
+        responseData.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData<>(Errors.NONE, 100000,
                 FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, abortedTransactions, records));
-        responseData.put(new TopicPartition("bar", 1), new FetchResponse.PartitionData(Errors.NONE, 900000,
+        responseData.put(new TopicPartition("bar", 1), new FetchResponse.PartitionData<>(Errors.NONE, 900000,
                 5, FetchResponse.INVALID_LOG_START_OFFSET, null, records));
-        responseData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData(Errors.NONE, 70000,
-                6, FetchResponse.INVALID_LOG_START_OFFSET, Collections.emptyList(), records));
+        responseData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData<>(Errors.NONE, 70000,
+                6, FetchResponse.INVALID_LOG_START_OFFSET, Collections.emptyList(), records));
 
-        FetchResponse response = new FetchResponse(Errors.NONE, responseData, 10, INVALID_SESSION_ID);
+        FetchResponse response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID);
         FetchResponse deserialized = FetchResponse.parse(toBuffer(response.toStruct((short) 4)), (short) 4);
         assertEquals(responseData, deserialized.responseData());
     }
@@ -595,7 +596,7 @@ public void testFetchRequestWithMetadata() throws Exception {
     }
 
     @Test
-    public void testJoinGroupRequestVersion0RebalanceTimeout() throws Exception {
+    public void testJoinGroupRequestVersion0RebalanceTimeout() {
         final short version = 0;
         JoinGroupRequest jgr = createJoinGroupRequest(version);
         JoinGroupRequest jgr2 = new JoinGroupRequest(jgr.toStruct(), version);
@@ -627,56 +628,61 @@ private FindCoordinatorResponse createFindCoordinatorResponse() {
 
     private FetchRequest createFetchRequest(int version, FetchMetadata metadata, List toForget) {
         LinkedHashMap fetchData = new LinkedHashMap<>();
-        fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L, 1000000));
-        fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L, 1000000));
+        fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L,
+                1000000, Optional.of(15)));
+        fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L,
+                1000000, Optional.of(25)));
         return FetchRequest.Builder.forConsumer(100, 100000, fetchData).
             metadata(metadata).setMaxBytes(1000).toForget(toForget).build((short) version);
     }
 
     private FetchRequest createFetchRequest(int version, IsolationLevel isolationLevel) {
         LinkedHashMap fetchData = new LinkedHashMap<>();
-        fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L, 1000000));
-        fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L, 1000000));
+        fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L,
+                1000000, Optional.of(15)));
+        fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L,
+                1000000, Optional.of(25)));
         return FetchRequest.Builder.forConsumer(100, 100000, fetchData).
             isolationLevel(isolationLevel).setMaxBytes(1000).build((short) version);
     }
 
     private FetchRequest createFetchRequest(int version) {
         LinkedHashMap fetchData = new LinkedHashMap<>();
-        fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L, 1000000));
-        fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L, 1000000));
+        fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 0L,
+                1000000, Optional.of(15)));
+        fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 0L,
+                1000000, Optional.of(25)));
         return FetchRequest.Builder.forConsumer(100, 100000, fetchData).setMaxBytes(1000).build((short) version);
     }
 
-    private FetchResponse createFetchResponse(Errors error, int sessionId) {
-        return new FetchResponse(error, new LinkedHashMap(),
-            25, sessionId);
+    private FetchResponse createFetchResponse(Errors error, int sessionId) {
+        return new FetchResponse<>(error, new LinkedHashMap<>(), 25, sessionId);
     }
 
-    private FetchResponse createFetchResponse(int sessionId) {
-        LinkedHashMap responseData = new LinkedHashMap<>();
+    private FetchResponse createFetchResponse(int sessionId) {
+        LinkedHashMap> responseData = new LinkedHashMap<>();
         MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes()));
-        responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE,
+        responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(Errors.NONE,
             1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records));
         List abortedTransactions = Collections.singletonList(
             new FetchResponse.AbortedTransaction(234L, 999L));
-        responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData(Errors.NONE,
+        responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData<>(Errors.NONE,
             1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, abortedTransactions, MemoryRecords.EMPTY));
-        return new FetchResponse(Errors.NONE, responseData, 25, sessionId);
+        return new FetchResponse<>(Errors.NONE, responseData, 25, sessionId);
     }
 
-    private FetchResponse createFetchResponse() {
-        LinkedHashMap responseData = new LinkedHashMap<>();
+    private FetchResponse createFetchResponse() {
+        LinkedHashMap> responseData = new LinkedHashMap<>();
         MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes()));
-        responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE,
+        responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(Errors.NONE,
                 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records));
 
         List abortedTransactions = Collections.singletonList(
                 new FetchResponse.AbortedTransaction(234L, 999L));
-        responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData(Errors.NONE,
+        responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData<>(Errors.NONE,
                 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, abortedTransactions, MemoryRecords.EMPTY));
 
-        return new FetchResponse(Errors.NONE, responseData, 25, INVALID_SESSION_ID);
+        return new FetchResponse<>(Errors.NONE, responseData, 25, INVALID_SESSION_ID);
     }
 
     private HeartbeatRequest createHeartBeatRequest() {
@@ -757,18 +763,20 @@ private ListOffsetRequest createListOffsetRequest(int version) {
                     new ListOffsetRequest.PartitionData(1000000L, 10));
             return ListOffsetRequest.Builder
                     .forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
-                    .setOffsetData(offsetData)
+                    .setTargetTimes(offsetData)
                     .build((short) version);
         } else if (version == 1) {
-            Map offsetData = Collections.singletonMap(
-                    new TopicPartition("test", 0), 1000000L);
+            Map offsetData = Collections.singletonMap(
+                    new TopicPartition("test", 0),
+                    new ListOffsetRequest.PartitionData(1000000L, Optional.empty()));
             return ListOffsetRequest.Builder
                     .forConsumer(true, IsolationLevel.READ_UNCOMMITTED)
                     .setTargetTimes(offsetData)
                     .build((short) version);
         } else if (version == 2) {
-            Map offsetData = Collections.singletonMap(
-                    new TopicPartition("test", 0), 1000000L);
+            Map offsetData = Collections.singletonMap(
+                    new TopicPartition("test", 0),
+                    new ListOffsetRequest.PartitionData(1000000L, Optional.of(5)));
             return ListOffsetRequest.Builder
                     .forConsumer(true, IsolationLevel.READ_COMMITTED)
                     .setTargetTimes(offsetData)
@@ -788,7 +796,7 @@ private ListOffsetResponse createListOffsetResponse(int version) {
         } else if (version == 1 || version == 2) {
             Map responseData = new HashMap<>();
             responseData.put(new TopicPartition("test", 0),
-                    new ListOffsetResponse.PartitionData(Errors.NONE, 10000L, 100L));
+                    new ListOffsetResponse.PartitionData(Errors.NONE, 10000L, 100L, Optional.of(27)));
             return new ListOffsetResponse(responseData);
         } else {
             throw new IllegalArgumentException("Illegal ListOffsetResponse version " + version);
@@ -807,20 +815,23 @@ private MetadataResponse createMetadataResponse() {
 
         List allTopicMetadata = new ArrayList<>();
         allTopicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE, "__consumer_offsets", true,
-                asList(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, node, replicas, isr, offlineReplicas))));
+                asList(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, node,
+                        Optional.of(5), replicas, isr, offlineReplicas))));
         allTopicMetadata.add(new MetadataResponse.TopicMetadata(Errors.LEADER_NOT_AVAILABLE, "topic2", false,
-                Collections.emptyList()));
+                Collections.emptyList()));
         allTopicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE, "topic3", false,
             asList(new MetadataResponse.PartitionMetadata(Errors.LEADER_NOT_AVAILABLE, 0, null,
-                replicas, isr, offlineReplicas))));
+                Optional.empty(), replicas, isr, offlineReplicas))));
 
         return new MetadataResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata);
     }
 
     private OffsetCommitRequest createOffsetCommitRequest(int version) {
         Map commitData = new HashMap<>();
-        commitData.put(new TopicPartition("test", 0), new OffsetCommitRequest.PartitionData(100, ""));
-        commitData.put(new TopicPartition("test", 1), new OffsetCommitRequest.PartitionData(200, null));
+        commitData.put(new TopicPartition("test", 0), new OffsetCommitRequest.PartitionData(100,
+                RecordBatch.NO_PARTITION_LEADER_EPOCH, ""));
+        commitData.put(new TopicPartition("test", 1), new OffsetCommitRequest.PartitionData(200,
+                RecordBatch.NO_PARTITION_LEADER_EPOCH, null));
         return new OffsetCommitRequest.Builder("group1", commitData)
                 .setGenerationId(100)
                 .setMemberId("consumer1")
@@ -840,8 +851,10 @@ private OffsetFetchRequest createOffsetFetchRequest(int version) {
 
     private OffsetFetchResponse createOffsetFetchResponse() {
         Map responseData = new HashMap<>();
-        responseData.put(new TopicPartition("test", 0), new OffsetFetchResponse.PartitionData(100L, "", Errors.NONE));
-        responseData.put(new TopicPartition("test", 1), new OffsetFetchResponse.PartitionData(100L, null, Errors.NONE));
+        responseData.put(new TopicPartition("test", 0), new OffsetFetchResponse.PartitionData(
+                100L, Optional.empty(), "", Errors.NONE));
+        responseData.put(new TopicPartition("test", 1), new OffsetFetchResponse.PartitionData(
+                100L, Optional.of(10), null, Errors.NONE));
         return new OffsetFetchResponse(Errors.NONE, responseData);
     }
 
@@ -1021,11 +1034,14 @@ private InitProducerIdResponse createInitPidResponse() {
 
 
     private OffsetsForLeaderEpochRequest createLeaderEpochRequest() {
-        Map epochs = new HashMap<>();
+        Map epochs = new HashMap<>();
 
-        epochs.put(new TopicPartition("topic1", 0), 1);
-        epochs.put(new TopicPartition("topic1", 1), 1);
-        epochs.put(new TopicPartition("topic2", 2), 3);
+        epochs.put(new TopicPartition("topic1", 0),
+                new OffsetsForLeaderEpochRequest.PartitionData(Optional.of(0), 1));
+        epochs.put(new TopicPartition("topic1", 1),
+                new OffsetsForLeaderEpochRequest.PartitionData(Optional.of(0), 1));
+        epochs.put(new TopicPartition("topic2", 2),
+                new OffsetsForLeaderEpochRequest.PartitionData(Optional.empty(), 3));
 
         return new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(), epochs).build();
     }
@@ -1082,7 +1098,9 @@ private WriteTxnMarkersResponse createWriteTxnMarkersResponse() {
     private TxnOffsetCommitRequest createTxnOffsetCommitRequest() {
         final Map offsets = new HashMap<>();
         offsets.put(new TopicPartition("topic", 73),
-                    new TxnOffsetCommitRequest.CommittedOffset(100, null));
+                    new TxnOffsetCommitRequest.CommittedOffset(100, null, Optional.empty()));
+        offsets.put(new TopicPartition("topic", 74),
+                new TxnOffsetCommitRequest.CommittedOffset(100, "blah", Optional.of(27)));
         return new TxnOffsetCommitRequest.Builder("transactionalId", "groupId", 21L, (short) 42, offsets).build();
     }
 
diff --git a/clients/src/test/resources/serializedData/offsetAndMetadataSerializedfile b/clients/src/test/resources/serializedData/offsetAndMetadataBeforeLeaderEpoch
similarity index 100%
rename from clients/src/test/resources/serializedData/offsetAndMetadataSerializedfile
rename to clients/src/test/resources/serializedData/offsetAndMetadataBeforeLeaderEpoch
diff --git a/clients/src/test/resources/serializedData/offsetAndMetadataWithLeaderEpoch b/clients/src/test/resources/serializedData/offsetAndMetadataWithLeaderEpoch
new file mode 100644
index 0000000000000000000000000000000000000000..ddf3956a0cee8b2745b8df0999ca00d1a1532402
GIT binary patch
literal 257
zcmZ4UmVvdnh`~6&C|xhHATc>3RWCa+Ejv*!IVUqUucTNnIX|zsG&i+K&p$1#IJLwv
zFU2>tBrzqiBvFR#*@vs)Hv*X$n7tU-^1;$R4BR=Xi7BZ?t_AtY86^zDK3R!niTXK-
zdFlF|c_pdosYTX43>>*oBT5*AkQ4-$6lLb6TUQh?KmZreVS*t2dLaFJQ2j!W7A*PR
z*lxkZ!05@qT9%konp#oBAb?QkSDKrYTGX~?sx0@E2i;5z3}6FamoSKwq!yPbB
-        (topicPartition, new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET, "", Errors.NONE))
+        val partitionData = new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET,
+          Optional.empty(), "", Errors.NONE)
+        topicPartition -> partitionData
       }.toMap
     } else {
       group.inLock {
         if (group.is(Dead)) {
           topicPartitionsOpt.getOrElse(Seq.empty[TopicPartition]).map { topicPartition =>
-            (topicPartition, new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET, "", Errors.NONE))
+            val partitionData = new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET,
+              Optional.empty(), "", Errors.NONE)
+            topicPartition -> partitionData
           }.toMap
         } else {
           topicPartitionsOpt match {
@@ -454,16 +459,19 @@ class GroupMetadataManager(brokerId: Int,
               // Return offsets for all partitions owned by this consumer group. (this only applies to consumers
               // that commit offsets to Kafka.)
               group.allOffsets.map { case (topicPartition, offsetAndMetadata) =>
-                topicPartition -> new OffsetFetchResponse.PartitionData(offsetAndMetadata.offset, offsetAndMetadata.metadata, Errors.NONE)
+                topicPartition -> new OffsetFetchResponse.PartitionData(offsetAndMetadata.offset,
+                  Optional.empty(), offsetAndMetadata.metadata, Errors.NONE)
               }
 
             case Some(topicPartitions) =>
               topicPartitions.map { topicPartition =>
                 val partitionData = group.offset(topicPartition) match {
                   case None =>
-                    new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET, "", Errors.NONE)
+                    new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET,
+                      Optional.empty(), "", Errors.NONE)
                   case Some(offsetAndMetadata) =>
-                    new OffsetFetchResponse.PartitionData(offsetAndMetadata.offset, offsetAndMetadata.metadata, Errors.NONE)
+                    new OffsetFetchResponse.PartitionData(offsetAndMetadata.offset,
+                      Optional.empty(), offsetAndMetadata.metadata, Errors.NONE)
                 }
                 topicPartition -> partitionData
               }.toMap
diff --git a/core/src/main/scala/kafka/server/FetchSession.scala b/core/src/main/scala/kafka/server/FetchSession.scala
index 64bc773f0bbb8..16ee872824610 100644
--- a/core/src/main/scala/kafka/server/FetchSession.scala
+++ b/core/src/main/scala/kafka/server/FetchSession.scala
@@ -18,6 +18,7 @@
 package kafka.server
 
 import java.util
+import java.util.Optional
 import java.util.concurrent.{ThreadLocalRandom, TimeUnit}
 
 import com.yammer.metrics.core.Gauge
@@ -107,7 +108,7 @@ class CachedPartition(val topic: String,
 
   def topicPartition = new TopicPartition(topic, partition)
 
-  def reqData = new FetchRequest.PartitionData(fetchOffset, fetcherLogStartOffset, maxBytes)
+  def reqData = new FetchRequest.PartitionData(fetchOffset, fetcherLogStartOffset, maxBytes, Optional.empty())
 
   def updateRequestParams(reqData: FetchRequest.PartitionData): Unit = {
     // Update our cached request parameters.
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 3a81b89f61a9e..24c13cd0c2f62 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -22,7 +22,7 @@ import java.nio.ByteBuffer
 import java.util
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.AtomicInteger
-import java.util.{Collections, Properties}
+import java.util.{Collections, Optional, Properties}
 
 import kafka.admin.{AdminUtils, RackAwareMode}
 import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0}
@@ -735,7 +735,7 @@ class KafkaApis(val requestChannel: RequestChannel,
     val clientId = request.header.clientId
     val offsetRequest = request.body[ListOffsetRequest]
 
-    val (authorizedRequestInfo, unauthorizedRequestInfo) = offsetRequest.offsetData.asScala.partition {
+    val (authorizedRequestInfo, unauthorizedRequestInfo) = offsetRequest.partitionTimestamps.asScala.partition {
       case (topicPartition, _) => authorize(request.session, Describe, Resource(Topic, topicPartition.topic, LITERAL))
     }
 
@@ -794,17 +794,19 @@ class KafkaApis(val requestChannel: RequestChannel,
 
     val unauthorizedResponseStatus = unauthorizedRequestInfo.mapValues(_ => {
       new ListOffsetResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED,
-                                           ListOffsetResponse.UNKNOWN_TIMESTAMP,
-                                           ListOffsetResponse.UNKNOWN_OFFSET)
+        ListOffsetResponse.UNKNOWN_TIMESTAMP,
+        ListOffsetResponse.UNKNOWN_OFFSET,
+        Optional.empty())
     })
 
-    val responseMap = authorizedRequestInfo.map { case (topicPartition, timestamp) =>
+    val responseMap = authorizedRequestInfo.map { case (topicPartition, partitionData) =>
       if (offsetRequest.duplicatePartitions().contains(topicPartition)) {
         debug(s"OffsetRequest with correlation id $correlationId from client $clientId on partition $topicPartition " +
             s"failed because the partition is duplicated in the request.")
         (topicPartition, new ListOffsetResponse.PartitionData(Errors.INVALID_REQUEST,
-                                                              ListOffsetResponse.UNKNOWN_TIMESTAMP,
-                                                              ListOffsetResponse.UNKNOWN_OFFSET))
+          ListOffsetResponse.UNKNOWN_TIMESTAMP,
+          ListOffsetResponse.UNKNOWN_OFFSET,
+          Optional.empty()))
       } else {
         try {
           // ensure leader exists
@@ -820,21 +822,22 @@ class KafkaApis(val requestChannel: RequestChannel,
               case IsolationLevel.READ_UNCOMMITTED => localReplica.highWatermark.messageOffset
             }
 
-            if (timestamp == ListOffsetRequest.LATEST_TIMESTAMP)
+            if (partitionData.timestamp == ListOffsetRequest.LATEST_TIMESTAMP)
               TimestampOffset(RecordBatch.NO_TIMESTAMP, lastFetchableOffset)
             else {
               def allowed(timestampOffset: TimestampOffset): Boolean =
-                timestamp == ListOffsetRequest.EARLIEST_TIMESTAMP || timestampOffset.offset < lastFetchableOffset
+                partitionData.timestamp == ListOffsetRequest.EARLIEST_TIMESTAMP || timestampOffset.offset < lastFetchableOffset
 
-              fetchOffsetForTimestamp(topicPartition, timestamp)
+              fetchOffsetForTimestamp(topicPartition, partitionData.timestamp)
                 .filter(allowed).getOrElse(TimestampOffset.Unknown)
             }
           } else {
-            fetchOffsetForTimestamp(topicPartition, timestamp)
+            fetchOffsetForTimestamp(topicPartition, partitionData.timestamp)
               .getOrElse(TimestampOffset.Unknown)
           }
 
-          (topicPartition, new ListOffsetResponse.PartitionData(Errors.NONE, found.timestamp, found.offset))
+          (topicPartition, new ListOffsetResponse.PartitionData(Errors.NONE, found.timestamp, found.offset,
+            Optional.empty()))
         } catch {
           // NOTE: These exceptions are special cased since these error messages are typically transient or the client
           // would have received a clear exception and there is no value in logging the entire stack trace for the same
@@ -845,13 +848,15 @@ class KafkaApis(val requestChannel: RequestChannel,
             debug(s"Offset request with correlation id $correlationId from client $clientId on " +
                 s"partition $topicPartition failed due to ${e.getMessage}")
             (topicPartition, new ListOffsetResponse.PartitionData(Errors.forException(e),
-                                                                  ListOffsetResponse.UNKNOWN_TIMESTAMP,
-                                                                  ListOffsetResponse.UNKNOWN_OFFSET))
+              ListOffsetResponse.UNKNOWN_TIMESTAMP,
+              ListOffsetResponse.UNKNOWN_OFFSET,
+              Optional.empty()))
           case e: Throwable =>
             error("Error while responding to offset request", e)
             (topicPartition, new ListOffsetResponse.PartitionData(Errors.forException(e),
-                                                                  ListOffsetResponse.UNKNOWN_TIMESTAMP,
-                                                                  ListOffsetResponse.UNKNOWN_OFFSET))
+              ListOffsetResponse.UNKNOWN_TIMESTAMP,
+              ListOffsetResponse.UNKNOWN_OFFSET,
+              Optional.empty()))
         }
       }
     }
@@ -1118,16 +1123,16 @@ class KafkaApis(val requestChannel: RequestChannel,
                   val payloadOpt = zkClient.getConsumerOffset(offsetFetchRequest.groupId, topicPartition)
                   payloadOpt match {
                     case Some(payload) =>
-                      (topicPartition, new OffsetFetchResponse.PartitionData(
-                          payload.toLong, OffsetFetchResponse.NO_METADATA, Errors.NONE))
+                      (topicPartition, new OffsetFetchResponse.PartitionData(payload.toLong,
+                        Optional.empty(), OffsetFetchResponse.NO_METADATA, Errors.NONE))
                     case None =>
                       (topicPartition, OffsetFetchResponse.UNKNOWN_PARTITION)
                   }
                 }
               } catch {
                 case e: Throwable =>
-                  (topicPartition, new OffsetFetchResponse.PartitionData(
-                      OffsetFetchResponse.INVALID_OFFSET, OffsetFetchResponse.NO_METADATA, Errors.forException(e)))
+                  (topicPartition, new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET,
+                    Optional.empty(), OffsetFetchResponse.NO_METADATA, Errors.forException(e)))
               }
             }.toMap
 
diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala
index 25967b306372c..31146636ae093 100755
--- a/core/src/main/scala/kafka/server/MetadataCache.scala
+++ b/core/src/main/scala/kafka/server/MetadataCache.scala
@@ -17,7 +17,7 @@
 
 package kafka.server
 
-import java.util.Collections
+import java.util.{Collections, Optional}
 import java.util.concurrent.locks.ReentrantReadWriteLock
 
 import scala.collection.{Seq, Set, mutable}
@@ -74,6 +74,7 @@ class MetadataCache(brokerId: Int) extends Logging {
       partitions.map { case (partitionId, partitionState) =>
         val topicPartition = new TopicPartition(topic, partitionId.toInt)
         val leaderBrokerId = partitionState.basePartitionState.leader
+        val leaderEpoch = partitionState.basePartitionState.leaderEpoch
         val maybeLeader = getAliveEndpoint(snapshot, leaderBrokerId, listenerName)
         val replicas = partitionState.basePartitionState.replicas.asScala.map(_.toInt)
         val replicaInfo = getEndpoints(snapshot, replicas, listenerName, errorUnavailableEndpoints)
@@ -89,7 +90,8 @@ class MetadataCache(brokerId: Int) extends Logging {
               if (errorUnavailableListeners) Errors.LISTENER_NOT_FOUND else Errors.LEADER_NOT_AVAILABLE
             }
             new MetadataResponse.PartitionMetadata(error, partitionId.toInt, Node.noNode(),
-              replicaInfo.asJava, java.util.Collections.emptyList(), offlineReplicaInfo.asJava)
+              Optional.empty(), replicaInfo.asJava, java.util.Collections.emptyList(),
+              offlineReplicaInfo.asJava)
 
           case Some(leader) =>
             val isr = partitionState.basePartitionState.isr.asScala.map(_.toInt)
@@ -100,15 +102,15 @@ class MetadataCache(brokerId: Int) extends Logging {
                 s"following brokers ${replicas.filterNot(replicaInfo.map(_.id).contains).mkString(",")}")
 
               new MetadataResponse.PartitionMetadata(Errors.REPLICA_NOT_AVAILABLE, partitionId.toInt, leader,
-                replicaInfo.asJava, isrInfo.asJava, offlineReplicaInfo.asJava)
+                Optional.empty(), replicaInfo.asJava, isrInfo.asJava, offlineReplicaInfo.asJava)
             } else if (isrInfo.size < isr.size) {
               debug(s"Error while fetching metadata for $topicPartition: in sync replica information not available for " +
                 s"following brokers ${isr.filterNot(isrInfo.map(_.id).contains).mkString(",")}")
               new MetadataResponse.PartitionMetadata(Errors.REPLICA_NOT_AVAILABLE, partitionId.toInt, leader,
-                replicaInfo.asJava, isrInfo.asJava, offlineReplicaInfo.asJava)
+                Optional.empty(), replicaInfo.asJava, isrInfo.asJava, offlineReplicaInfo.asJava)
             } else {
-              new MetadataResponse.PartitionMetadata(Errors.NONE, partitionId.toInt, leader, replicaInfo.asJava,
-                isrInfo.asJava, offlineReplicaInfo.asJava)
+              new MetadataResponse.PartitionMetadata(Errors.NONE, partitionId.toInt, leader, Optional.of(leaderEpoch),
+                replicaInfo.asJava, isrInfo.asJava, offlineReplicaInfo.asJava)
             }
         }
       }
diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
index 5aec7a90d2922..dc585ebd9260d 100644
--- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala
@@ -18,6 +18,7 @@
 package kafka.server
 
 import java.util
+import java.util.Optional
 
 import kafka.api.Request
 import kafka.cluster.BrokerEndPoint
@@ -196,7 +197,8 @@ class ReplicaAlterLogDirsThread(name: String,
       val (topicPartition, partitionFetchState) = maxPartitionOpt.get
       try {
         val logStartOffset = replicaMgr.getReplicaOrException(topicPartition, Request.FutureLocalReplicaId).logStartOffset
-        requestMap.put(topicPartition, new FetchRequest.PartitionData(partitionFetchState.fetchOffset, logStartOffset, fetchSize))
+        requestMap.put(topicPartition, new FetchRequest.PartitionData(partitionFetchState.fetchOffset, logStartOffset,
+          fetchSize, Optional.empty()))
       } catch {
         case _: KafkaStorageException =>
           partitionsWithError += topicPartition
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index 1848eb741c381..5dcd29b473d26 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -17,6 +17,8 @@
 
 package kafka.server
 
+import java.util.Optional
+
 import kafka.api._
 import kafka.cluster.BrokerEndPoint
 import kafka.log.{LogAppendInfo, LogConfig}
@@ -61,7 +63,8 @@ class ReplicaFetcherThread(name: String,
 
   // Visible for testing
   private[server] val fetchRequestVersion: Short =
-    if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1) 8
+    if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_1_IV1) 9
+    else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1) 8
     else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_1_1_IV0) 7
     else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV1) 5
     else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV0) 4
@@ -72,12 +75,14 @@ class ReplicaFetcherThread(name: String,
 
   // Visible for testing
   private[server] val offsetForLeaderEpochRequestVersion: Short =
-    if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV0) 1
+    if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_1_IV1) 2
+    else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV0) 1
     else 0
 
   // Visible for testing
   private[server] val listOffsetRequestVersion: Short =
-    if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1) 3
+    if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_1_IV1) 4
+    else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_2_0_IV1) 3
     else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_11_0_IV0) 2
     else if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_1_IV2) 1
     else 0
@@ -199,22 +204,21 @@ class ReplicaFetcherThread(name: String,
   }
 
   private def fetchOffsetFromLeader(topicPartition: TopicPartition, earliestOrLatest: Long): Long = {
-    val requestBuilder = if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_1_IV2) {
-        val partitions = Map(topicPartition -> (earliestOrLatest: java.lang.Long))
-        ListOffsetRequest.Builder.forReplica(listOffsetRequestVersion, replicaId).setTargetTimes(partitions.asJava)
-      } else {
-        val partitions = Map(topicPartition -> new ListOffsetRequest.PartitionData(earliestOrLatest, 1))
-        ListOffsetRequest.Builder.forReplica(listOffsetRequestVersion, replicaId).setOffsetData(partitions.asJava)
-      }
+    val requestPartitionData = new ListOffsetRequest.PartitionData(earliestOrLatest, Optional.empty[Integer]())
+    val requestPartitions = Map(topicPartition -> requestPartitionData)
+    val requestBuilder = ListOffsetRequest.Builder.forReplica(listOffsetRequestVersion, replicaId)
+      .setTargetTimes(requestPartitions.asJava)
+
     val clientResponse = leaderEndpoint.sendRequest(requestBuilder)
     val response = clientResponse.responseBody.asInstanceOf[ListOffsetResponse]
-    val partitionData = response.responseData.get(topicPartition)
-    partitionData.error match {
+
+    val responsePartitionData = response.responseData.get(topicPartition)
+    responsePartitionData.error match {
       case Errors.NONE =>
         if (brokerConfig.interBrokerProtocolVersion >= KAFKA_0_10_1_IV2)
-          partitionData.offset
+          responsePartitionData.offset
         else
-          partitionData.offsets.get(0)
+          responsePartitionData.offsets.get(0)
       case error => throw error.exception
     }
   }
@@ -229,7 +233,7 @@ class ReplicaFetcherThread(name: String,
         try {
           val logStartOffset = replicaMgr.getReplicaOrException(topicPartition).logStartOffset
           builder.add(topicPartition, new FetchRequest.PartitionData(
-            partitionFetchState.fetchOffset, logStartOffset, fetchSize))
+            partitionFetchState.fetchOffset, logStartOffset, fetchSize, Optional.empty()))
         } catch {
           case _: KafkaStorageException =>
             // The replica has already been marked offline due to log directory failure and the original failure should have already been logged.
@@ -289,7 +293,9 @@ class ReplicaFetcherThread(name: String,
         return resultWithoutEpoch
       }
 
-      val partitionsAsJava = partitionsWithEpoch.map { case (tp, epoch) => tp -> epoch.asInstanceOf[Integer] }.toMap.asJava
+      val partitionsAsJava = partitions.map { case (tp, epoch) => tp ->
+        new OffsetsForLeaderEpochRequest.PartitionData(Optional.empty(), epoch.asInstanceOf[Integer])
+      }.toMap.asJava
       val epochRequest = new OffsetsForLeaderEpochRequest.Builder(offsetForLeaderEpochRequestVersion, partitionsAsJava)
       try {
         val response = leaderEndpoint.sendRequest(epochRequest)
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 2393daad003fa..d61240e3af680 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -1475,14 +1475,14 @@ class ReplicaManager(val config: KafkaConfig,
     new ReplicaAlterLogDirsManager(config, this, quotaManager, brokerTopicStats)
   }
 
-  def lastOffsetForLeaderEpoch(requestedEpochInfo: Map[TopicPartition, Integer]): Map[TopicPartition, EpochEndOffset] = {
-    requestedEpochInfo.map { case (tp, leaderEpoch) =>
+  def lastOffsetForLeaderEpoch(requestedEpochInfo: Map[TopicPartition, OffsetsForLeaderEpochRequest.PartitionData]): Map[TopicPartition, EpochEndOffset] = {
+    requestedEpochInfo.map { case (tp, partitionData) =>
       val epochEndOffset = getPartition(tp) match {
         case Some(partition) =>
           if (partition eq ReplicaManager.OfflinePartition)
             new EpochEndOffset(Errors.KAFKA_STORAGE_ERROR, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
           else
-            partition.lastOffsetForLeaderEpoch(leaderEpoch)
+            partition.lastOffsetForLeaderEpoch(partitionData.leaderEpoch)
 
         case None if metadataCache.contains(tp) =>
           new EpochEndOffset(Errors.NOT_LEADER_FOR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET)
diff --git a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala
index 1ecea09b9946d..4758764be652a 100644
--- a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala
+++ b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala
@@ -23,7 +23,7 @@ import java.util
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
 import java.util.regex.{Pattern, PatternSyntaxException}
-import java.util.{Date, Properties}
+import java.util.{Date, Optional, Properties}
 
 import joptsimple.OptionParser
 import kafka.api._
@@ -389,7 +389,8 @@ private class ReplicaFetcher(name: String, sourceBroker: Node, topicPartitions:
 
     val requestMap = new util.LinkedHashMap[TopicPartition, JFetchRequest.PartitionData]
     for (topicPartition <- topicPartitions)
-      requestMap.put(topicPartition, new JFetchRequest.PartitionData(replicaBuffer.getOffset(topicPartition), 0L, fetchSize))
+      requestMap.put(topicPartition, new JFetchRequest.PartitionData(replicaBuffer.getOffset(topicPartition),
+        0L, fetchSize, Optional.empty()))
 
     val fetchRequestBuilder = JFetchRequest.Builder.
       forReplica(ApiKeys.FETCH.latestVersion, Request.DebuggingConsumerId, maxWait, minBytes, requestMap)
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 7c341e6fb9b62..cfc6b5e43d751 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -16,7 +16,7 @@ import java.nio.ByteBuffer
 import java.util
 import java.util.concurrent.ExecutionException
 import java.util.regex.Pattern
-import java.util.{ArrayList, Collections, Properties}
+import java.util.{ArrayList, Collections, Optional, Properties}
 import java.time.Duration
 
 import kafka.admin.ConsumerGroupCommand.{ConsumerGroupCommandOptions, ConsumerGroupService}
@@ -260,25 +260,26 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   private def createFetchRequest = {
     val partitionMap = new util.LinkedHashMap[TopicPartition, requests.FetchRequest.PartitionData]
-    partitionMap.put(tp, new requests.FetchRequest.PartitionData(0, 0, 100))
+    partitionMap.put(tp, new requests.FetchRequest.PartitionData(0, 0, 100, Optional.of(27)))
     requests.FetchRequest.Builder.forConsumer(100, Int.MaxValue, partitionMap).build()
   }
 
   private def createFetchFollowerRequest = {
     val partitionMap = new util.LinkedHashMap[TopicPartition, requests.FetchRequest.PartitionData]
-    partitionMap.put(tp, new requests.FetchRequest.PartitionData(0, 0, 100))
+    partitionMap.put(tp, new requests.FetchRequest.PartitionData(0, 0, 100, Optional.of(27)))
     val version = ApiKeys.FETCH.latestVersion
     requests.FetchRequest.Builder.forReplica(version, 5000, 100, Int.MaxValue, partitionMap).build()
   }
 
   private def createListOffsetsRequest = {
     requests.ListOffsetRequest.Builder.forConsumer(false, IsolationLevel.READ_UNCOMMITTED).setTargetTimes(
-      Map(tp -> (0L: java.lang.Long)).asJava).
+      Map(tp -> new ListOffsetRequest.PartitionData(0L, Optional.of[Integer](27))).asJava).
       build()
   }
 
   private def offsetsForLeaderEpochRequest = {
-    new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion()).add(tp, 7).build()
+    new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion)
+      .add(tp, Optional.of(27), 7).build()
   }
 
   private def createOffsetFetchRequest = {
@@ -316,7 +317,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
 
   private def createOffsetCommitRequest = {
     new requests.OffsetCommitRequest.Builder(
-      group, Map(tp -> new requests.OffsetCommitRequest.PartitionData(0, "metadata")).asJava).
+      group, Map(tp -> new requests.OffsetCommitRequest.PartitionData(0, 27, "metadata")).asJava).
       setMemberId("").setGenerationId(1).
       build()
   }
diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
index 2befc8fdf3b0b..d2d115b2cfaa7 100644
--- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
+++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala
@@ -80,8 +80,9 @@ class ApiVersionTest {
     assertEquals(KAFKA_2_0_IV0, ApiVersion("2.0-IV0"))
     assertEquals(KAFKA_2_0_IV1, ApiVersion("2.0-IV1"))
 
-    assertEquals(KAFKA_2_1_IV0, ApiVersion("2.1"))
+    assertEquals(KAFKA_2_1_IV1, ApiVersion("2.1"))
     assertEquals(KAFKA_2_1_IV0, ApiVersion("2.1-IV0"))
+    assertEquals(KAFKA_2_1_IV1, ApiVersion("2.1-IV1"))
   }
 
   @Test
diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
index c456433b6e359..8c1d95a4356e4 100644
--- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala
@@ -18,6 +18,7 @@
 package kafka.server
 
 import java.nio.ByteBuffer
+import java.util.Optional
 
 import AbstractFetcherThread._
 import com.yammer.metrics.Metrics
@@ -371,7 +372,8 @@ class AbstractFetcherThreadTest {
       partitionMap.foreach { case (partition, state) =>
         if (state.isReadyForFetch) {
           val replicaState = replicaPartitionState(partition)
-          fetchData.put(partition, new FetchRequest.PartitionData(state.fetchOffset, replicaState.logStartOffset, 1024 * 1024))
+          fetchData.put(partition, new FetchRequest.PartitionData(state.fetchOffset, replicaState.logStartOffset,
+            1024 * 1024, Optional.empty()))
         }
       }
       val fetchRequest = FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion, replicaId, 0, 1, fetchData.asJava)
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala
index 1bf6f28eec515..673abe693c9c8 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala
@@ -17,7 +17,7 @@
 package kafka.server
 
 import java.util
-import java.util.Properties
+import java.util.{Optional, Properties}
 
 import kafka.log.LogConfig
 import kafka.utils.TestUtils
@@ -72,7 +72,8 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest {
                                  offsetMap: Map[TopicPartition, Long] = Map.empty): util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] = {
     val partitionMap = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
     topicPartitions.foreach { tp =>
-      partitionMap.put(tp, new FetchRequest.PartitionData(offsetMap.getOrElse(tp, 0), 0L, maxPartitionBytes))
+      partitionMap.put(tp, new FetchRequest.PartitionData(offsetMap.getOrElse(tp, 0), 0L,
+        maxPartitionBytes, Optional.empty()))
     }
     partitionMap
   }
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index 694b19d378ca0..f8c02cf5318c5 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -18,7 +18,7 @@ package kafka.server
 
 import java.io.DataInputStream
 import java.util
-import java.util.Properties
+import java.util.{Optional, Properties}
 
 import kafka.api.KAFKA_0_11_0_IV2
 import kafka.log.LogConfig
@@ -58,7 +58,8 @@ class FetchRequestTest extends BaseRequestTest {
                                  offsetMap: Map[TopicPartition, Long] = Map.empty): util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] = {
     val partitionMap = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
     topicPartitions.foreach { tp =>
-      partitionMap.put(tp, new FetchRequest.PartitionData(offsetMap.getOrElse(tp, 0), 0L, maxPartitionBytes))
+      partitionMap.put(tp, new FetchRequest.PartitionData(offsetMap.getOrElse(tp, 0), 0L, maxPartitionBytes,
+        Optional.empty()))
     }
     partitionMap
   }
diff --git a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala
index c4a96254eda28..74dae7c5303ba 100755
--- a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala
@@ -17,7 +17,7 @@
 package kafka.server
 
 import java.util
-import java.util.Collections
+import java.util.{Collections, Optional}
 
 import kafka.utils.MockTime
 import org.apache.kafka.common.TopicPartition
@@ -141,8 +141,10 @@ class FetchSessionTest {
 
     // Create a new fetch session with a FULL fetch request
     val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
-    reqData2.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100))
-    reqData2.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100))
+    reqData2.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100,
+      Optional.empty()))
+    reqData2.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100,
+      Optional.empty()))
     val context2 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData2, EMPTY_PART_LIST, false)
     assertEquals(classOf[FullFetchContext], context2.getClass)
     val reqData2Iter = reqData2.entrySet().iterator()
@@ -215,8 +217,10 @@ class FetchSessionTest {
     var nextSessionId = prevSessionId
     do {
       val reqData8 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
-      reqData8.put(new TopicPartition("bar", 0), new FetchRequest.PartitionData(0, 0, 100))
-      reqData8.put(new TopicPartition("bar", 1), new FetchRequest.PartitionData(10, 0, 100))
+      reqData8.put(new TopicPartition("bar", 0), new FetchRequest.PartitionData(0, 0, 100,
+        Optional.empty()))
+      reqData8.put(new TopicPartition("bar", 1), new FetchRequest.PartitionData(10, 0, 100,
+        Optional.empty()))
       val context8 = fetchManager.newContext(
         new JFetchMetadata(prevSessionId, FINAL_EPOCH), reqData8, EMPTY_PART_LIST, false)
       assertEquals(classOf[SessionlessFetchContext], context8.getClass)
@@ -240,8 +244,10 @@ class FetchSessionTest {
 
     // Create a new fetch session with foo-0 and foo-1
     val reqData1 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
-    reqData1.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100))
-    reqData1.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100))
+    reqData1.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100,
+      Optional.empty()))
+    reqData1.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100,
+      Optional.empty()))
     val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false)
     assertEquals(classOf[FullFetchContext], context1.getClass)
     val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]]
@@ -256,7 +262,8 @@ class FetchSessionTest {
 
     // Create an incremental fetch request that removes foo-0 and adds bar-0
     val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
-    reqData2.put(new TopicPartition("bar", 0), new FetchRequest.PartitionData(15, 0, 0))
+    reqData2.put(new TopicPartition("bar", 0), new FetchRequest.PartitionData(15, 0, 0,
+      Optional.empty()))
     val removed2 = new util.ArrayList[TopicPartition]
     removed2.add(new TopicPartition("foo", 0))
     val context2 = fetchManager.newContext(
@@ -290,8 +297,10 @@ class FetchSessionTest {
 
     // Create a new fetch session with foo-0 and foo-1
     val reqData1 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
-    reqData1.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100))
-    reqData1.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100))
+    reqData1.put(new TopicPartition("foo", 0), new FetchRequest.PartitionData(0, 0, 100,
+      Optional.empty()))
+    reqData1.put(new TopicPartition("foo", 1), new FetchRequest.PartitionData(10, 0, 100,
+      Optional.empty()))
     val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false)
     assertEquals(classOf[FullFetchContext], context1.getClass)
     val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]]
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 18b18fdc35540..eb3f28e60e7f7 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -20,7 +20,7 @@ package kafka.server
 import java.lang.{Long => JLong}
 import java.net.InetAddress
 import java.util
-import java.util.Collections
+import java.util.{Collections, Optional}
 
 import kafka.api.{ApiVersion, KAFKA_0_10_2_IV0}
 import kafka.cluster.Replica
@@ -118,7 +118,7 @@ class KafkaApisTest {
       EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel)
 
       val invalidTopicPartition = new TopicPartition(topic, invalidPartitionId)
-      val partitionOffsetCommitData = new OffsetCommitRequest.PartitionData(15L, "")
+      val partitionOffsetCommitData = new OffsetCommitRequest.PartitionData(15L, 23, "")
       val (offsetCommitRequest, request) = buildRequest(new OffsetCommitRequest.Builder("groupId",
         Map(invalidTopicPartition -> partitionOffsetCommitData).asJava))
 
@@ -144,7 +144,7 @@ class KafkaApisTest {
       EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel)
 
       val invalidTopicPartition = new TopicPartition(topic, invalidPartitionId)
-      val partitionOffsetCommitData = new TxnOffsetCommitRequest.CommittedOffset(15L, "")
+      val partitionOffsetCommitData = new TxnOffsetCommitRequest.CommittedOffset(15L, "", Optional.empty())
       val (offsetCommitRequest, request) = buildRequest(new TxnOffsetCommitRequest.Builder("txnlId", "groupId",
         15L, 0.toShort, Map(invalidTopicPartition -> partitionOffsetCommitData).asJava))
 
@@ -377,8 +377,10 @@ class KafkaApisTest {
     val capturedResponse = expectNoThrottling()
     EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, replica, log)
 
+
+    val targetTimes = Map(tp -> new ListOffsetRequest.PartitionData(timestamp, Optional.empty[Integer]()))
     val builder = ListOffsetRequest.Builder.forConsumer(true, isolationLevel)
-      .setTargetTimes(Map(tp -> timestamp).asJava)
+      .setTargetTimes(targetTimes.asJava)
     val (listOffsetRequest, request) = buildRequest(builder)
     createKafkaApis().handleListOffsetRequest(request)
 
@@ -418,8 +420,10 @@ class KafkaApisTest {
     val capturedResponse = expectNoThrottling()
     EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, replica, log)
 
+    val targetTimes = Map(tp -> new ListOffsetRequest.PartitionData(ListOffsetRequest.EARLIEST_TIMESTAMP,
+      Optional.empty[Integer]()))
     val builder = ListOffsetRequest.Builder.forConsumer(true, isolationLevel)
-      .setTargetTimes(Map(tp -> (ListOffsetRequest.EARLIEST_TIMESTAMP: JLong)).asJava)
+      .setTargetTimes(targetTimes.asJava)
     val (listOffsetRequest, request) = buildRequest(builder)
     createKafkaApis().handleListOffsetRequest(request)
 
@@ -508,8 +512,10 @@ class KafkaApisTest {
 
     EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, replica, log)
 
+    val targetTimes = Map(tp -> new ListOffsetRequest.PartitionData(ListOffsetRequest.LATEST_TIMESTAMP,
+      Optional.empty[Integer]()))
     val builder = ListOffsetRequest.Builder.forConsumer(true, isolationLevel)
-      .setTargetTimes(Map(tp -> (ListOffsetRequest.LATEST_TIMESTAMP: JLong)).asJava)
+      .setTargetTimes(targetTimes.asJava)
     val (listOffsetRequest, request) = buildRequest(builder)
     createKafkaApis().handleListOffsetRequest(request)
 
diff --git a/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala
index 6ee47eecda2ff..965413eb61225 100644
--- a/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala
@@ -16,7 +16,7 @@
  */
 package kafka.server
 
-import java.lang.{Long => JLong}
+import java.util.Optional
 
 import kafka.utils.TestUtils
 import org.apache.kafka.common.TopicPartition
@@ -33,20 +33,22 @@ class ListOffsetsRequestTest extends BaseRequestTest {
   def testListOffsetsErrorCodes(): Unit = {
     val topic = "topic"
     val partition = new TopicPartition(topic, 0)
+    val targetTimes = Map(partition -> new ListOffsetRequest.PartitionData(
+      ListOffsetRequest.EARLIEST_TIMESTAMP, Optional.of[Integer](0))).asJava
 
     val consumerRequest = ListOffsetRequest.Builder
       .forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
-      .setTargetTimes(Map(partition -> ListOffsetRequest.EARLIEST_TIMESTAMP.asInstanceOf[JLong]).asJava)
+      .setTargetTimes(targetTimes)
       .build()
 
     val replicaRequest = ListOffsetRequest.Builder
       .forReplica(ApiKeys.LIST_OFFSETS.latestVersion, servers.head.config.brokerId)
-      .setTargetTimes(Map(partition -> ListOffsetRequest.EARLIEST_TIMESTAMP.asInstanceOf[JLong]).asJava)
+      .setTargetTimes(targetTimes)
       .build()
 
     val debugReplicaRequest = ListOffsetRequest.Builder
       .forReplica(ApiKeys.LIST_OFFSETS.latestVersion, ListOffsetRequest.DEBUGGING_REPLICA_ID)
-      .setTargetTimes(Map(partition -> ListOffsetRequest.EARLIEST_TIMESTAMP.asInstanceOf[JLong]).asJava)
+      .setTargetTimes(targetTimes)
       .build()
 
     // Unknown topic
diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
index e371f7f587cad..45096cc61e3a2 100755
--- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala
@@ -19,7 +19,7 @@ package kafka.server
 
 import java.io.File
 import java.util.concurrent.atomic.AtomicInteger
-import java.util.{Properties, Random}
+import java.util.{Optional, Properties, Random}
 
 import kafka.log.{Log, LogSegment}
 import kafka.network.SocketServer
@@ -54,7 +54,7 @@ class LogOffsetTest extends BaseRequestTest {
   def testGetOffsetsForUnknownTopic() {
     val topicPartition = new TopicPartition("foo", 0)
     val request = ListOffsetRequest.Builder.forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
-      .setOffsetData(Map(topicPartition ->
+      .setTargetTimes(Map(topicPartition ->
         new ListOffsetRequest.PartitionData(ListOffsetRequest.LATEST_TIMESTAMP, 10)).asJava).build(0)
     val response = sendListOffsetsRequest(request)
     assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.responseData.get(topicPartition).error)
@@ -86,7 +86,7 @@ class LogOffsetTest extends BaseRequestTest {
     TestUtils.waitUntilTrue(() => TestUtils.isLeaderLocalOnBroker(topic, topicPartition.partition, server),
       "Leader should be elected")
     val request = ListOffsetRequest.Builder.forReplica(0, 0)
-      .setOffsetData(Map(topicPartition ->
+      .setTargetTimes(Map(topicPartition ->
         new ListOffsetRequest.PartitionData(ListOffsetRequest.LATEST_TIMESTAMP, 15)).asJava).build()
     val consumerOffsets = sendListOffsetsRequest(request).responseData.get(topicPartition).offsets.asScala
     assertEquals(Seq(20L, 18L, 16L, 14L, 12L, 10L, 8L, 6L, 4L, 3L), consumerOffsets)
@@ -114,7 +114,7 @@ class LogOffsetTest extends BaseRequestTest {
     TestUtils.waitUntilTrue(() => TestUtils.isLeaderLocalOnBroker(topic, topicPartition.partition, server),
       "Leader should be elected")
     val request = ListOffsetRequest.Builder.forReplica(0, 0)
-      .setOffsetData(Map(topicPartition ->
+      .setTargetTimes(Map(topicPartition ->
         new ListOffsetRequest.PartitionData(ListOffsetRequest.LATEST_TIMESTAMP, 15)).asJava).build()
     val consumerOffsets = sendListOffsetsRequest(request).responseData.get(topicPartition).offsets.asScala
     assertEquals(Seq(20L, 18L, 16L, 14L, 12L, 10L, 8L, 6L, 4L, 2L, 0L), consumerOffsets)
@@ -122,7 +122,7 @@ class LogOffsetTest extends BaseRequestTest {
     // try to fetch using latest offset
     val fetchRequest = FetchRequest.Builder.forConsumer(0, 1,
       Map(topicPartition -> new FetchRequest.PartitionData(consumerOffsets.head, FetchRequest.INVALID_LOG_START_OFFSET,
-        300 * 1024)).asJava).build()
+        300 * 1024, Optional.empty())).asJava).build()
     val fetchResponse = sendFetchRequest(fetchRequest)
     assertFalse(fetchResponse.responseData.get(topicPartition).records.batches.iterator.hasNext)
   }
@@ -142,7 +142,7 @@ class LogOffsetTest extends BaseRequestTest {
     for (_ <- 1 to 14) {
       val topicPartition = new TopicPartition(topic, 0)
       val request = ListOffsetRequest.Builder.forReplica(0, 0)
-        .setOffsetData(Map(topicPartition ->
+        .setTargetTimes(Map(topicPartition ->
           new ListOffsetRequest.PartitionData(ListOffsetRequest.EARLIEST_TIMESTAMP, 1)).asJava).build()
       val consumerOffsets = sendListOffsetsRequest(request).responseData.get(topicPartition).offsets.asScala
       if (consumerOffsets.head == 1)
@@ -174,7 +174,7 @@ class LogOffsetTest extends BaseRequestTest {
     TestUtils.waitUntilTrue(() => TestUtils.isLeaderLocalOnBroker(topic, topicPartition.partition, server),
       "Leader should be elected")
     val request = ListOffsetRequest.Builder.forReplica(0, 0)
-      .setOffsetData(Map(topicPartition ->
+      .setTargetTimes(Map(topicPartition ->
         new ListOffsetRequest.PartitionData(now, 15)).asJava).build()
     val consumerOffsets = sendListOffsetsRequest(request).responseData.get(topicPartition).offsets.asScala
     assertEquals(Seq(20L, 18L, 16L, 14L, 12L, 10L, 8L, 6L, 4L, 2L, 0L), consumerOffsets)
@@ -201,7 +201,7 @@ class LogOffsetTest extends BaseRequestTest {
     TestUtils.waitUntilTrue(() => TestUtils.isLeaderLocalOnBroker(topic, topicPartition.partition, server),
       "Leader should be elected")
     val request = ListOffsetRequest.Builder.forReplica(0, 0)
-      .setOffsetData(Map(topicPartition ->
+      .setTargetTimes(Map(topicPartition ->
         new ListOffsetRequest.PartitionData(ListOffsetRequest.EARLIEST_TIMESTAMP, 10)).asJava).build()
     val consumerOffsets = sendListOffsetsRequest(request).responseData.get(topicPartition).offsets.asScala
     assertEquals(Seq(0L), consumerOffsets)
diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
index 93ac62d562ded..ec3534f278611 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
@@ -17,6 +17,7 @@
 package kafka.server
 
 import java.util
+import java.util.Optional
 import util.Arrays.asList
 
 import org.apache.kafka.common.TopicPartition
@@ -45,7 +46,6 @@ class MetadataCacheTest {
     val topic0 = "topic-0"
     val topic1 = "topic-1"
 
-
     val cache = new MetadataCache(1)
 
     val zkVersion = 3
@@ -95,6 +95,7 @@ class MetadataCacheTest {
           val leader = partitionMetadata.leader
           val partitionState = topicPartitionStates(new TopicPartition(topic, partitionId))
           assertEquals(partitionState.basePartitionState.leader, leader.id)
+          assertEquals(Optional.of(partitionState.basePartitionState.leaderEpoch), partitionMetadata.leaderEpoch)
           assertEquals(partitionState.basePartitionState.isr, partitionMetadata.isr.asScala.map(_.id).asJava)
           assertEquals(partitionState.basePartitionState.replicas, partitionMetadata.replicas.asScala.map(_.id).asJava)
           val endPoint = endPoints(partitionMetadata.leader.id).find(_.listenerName == listenerName).get
diff --git a/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala b/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala
index c6385f325ece4..e314b443925d4 100644
--- a/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala
@@ -16,6 +16,8 @@
  */
 package kafka.server
 
+import java.util.Optional
+
 import kafka.utils.TestUtils
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.protocol.{ApiKeys, Errors}
@@ -33,7 +35,7 @@ class OffsetsForLeaderEpochRequestTest extends BaseRequestTest {
     val partition = new TopicPartition(topic, 0)
 
     val request = new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion)
-      .add(partition, 0)
+      .add(partition, Optional.of(5), 0)
       .build()
 
     // Unknown topic
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
index 9c759cebc3f87..520801c9babee 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala
@@ -486,7 +486,7 @@ class ReplicaFetcherThreadTest {
     assertEquals(2, mockNetwork.epochFetchCount)
     assertEquals(1, mockNetwork.fetchCount)
     assertEquals("OffsetsForLeaderEpochRequest version.",
-                 1, mockNetwork.lastUsedOffsetForLeaderEpochVersion)
+      2, mockNetwork.lastUsedOffsetForLeaderEpochVersion)
 
     //Loop 3 we should not fetch epochs
     thread.doWork()
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
index 66a2c8e5fec11..ea0a8ef800026 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
@@ -17,7 +17,7 @@
 package kafka.server
 
 import java.io.File
-import java.util.Properties
+import java.util.{Optional, Properties}
 import java.util.concurrent.atomic.AtomicBoolean
 
 import kafka.cluster.Replica
@@ -43,7 +43,9 @@ class ReplicaManagerQuotasTest {
   val record = new SimpleRecord("some-data-in-a-message".getBytes())
   val topicPartition1 = new TopicPartition("test-topic", 1)
   val topicPartition2 = new TopicPartition("test-topic", 2)
-  val fetchInfo = Seq(topicPartition1 -> new PartitionData(0, 0, 100), topicPartition2 -> new PartitionData(0, 0, 100))
+  val fetchInfo = Seq(
+    topicPartition1 -> new PartitionData(0, 0, 100, Optional.empty()),
+    topicPartition2 -> new PartitionData(0, 0, 100, Optional.empty()))
   var replicaManager: ReplicaManager = _
 
   @Test
@@ -164,9 +166,9 @@ class ReplicaManagerQuotasTest {
       EasyMock.replay(replicaManager)
 
       val tp = new TopicPartition("t1", 0)
-      val fetchParititonStatus = new FetchPartitionStatus(new LogOffsetMetadata(messageOffset = 50L, segmentBaseOffset = 0L,
-        relativePositionInSegment = 250), new PartitionData(50, 0, 1))
-      val fetchMetadata = new FetchMetadata(fetchMinBytes = 1, fetchMaxBytes = 1000, hardMaxBytesLimit = true, fetchOnlyLeader = true,
+      val fetchParititonStatus = FetchPartitionStatus(new LogOffsetMetadata(messageOffset = 50L, segmentBaseOffset = 0L,
+         relativePositionInSegment = 250), new PartitionData(50, 0, 1, Optional.empty()))
+      val fetchMetadata = FetchMetadata(fetchMinBytes = 1, fetchMaxBytes = 1000, hardMaxBytesLimit = true, fetchOnlyLeader = true,
         fetchOnlyCommitted = false, isFromFollower = true, replicaId = 1, fetchPartitionStatus = List((tp, fetchParititonStatus)))
       new DelayedFetch(delayMs = 600, fetchMetadata = fetchMetadata, replicaManager = replicaManager,
         quota = null, isolationLevel = IsolationLevel.READ_UNCOMMITTED, responseCallback = null) {
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 41564a54b0573..08440528638ac 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -18,7 +18,7 @@
 package kafka.server
 
 import java.io.File
-import java.util.Properties
+import java.util.{Optional, Properties}
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.concurrent.atomic.AtomicBoolean
 
@@ -162,7 +162,8 @@ class ReplicaManagerTest {
       partition.getOrCreateReplica(0)
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0,
-        collection.immutable.Map(new TopicPartition(topic, 0) -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, false)).asJava,
+        collection.immutable.Map(new TopicPartition(topic, 0) ->
+          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, false)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       rm.getLeaderReplicaIfLocal(new TopicPartition(topic, 0))
@@ -173,13 +174,15 @@ class ReplicaManagerTest {
       }
 
       // Fetch some messages
-      val fetchResult = fetchAsConsumer(rm, new TopicPartition(topic, 0), new PartitionData(0, 0, 100000),
+      val fetchResult = fetchAsConsumer(rm, new TopicPartition(topic, 0),
+        new PartitionData(0, 0, 100000, Optional.empty()),
         minBytes = 100000)
       assertFalse(fetchResult.isFired)
 
       // Make this replica the follower
       val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0,
-        collection.immutable.Map(new TopicPartition(topic, 0) -> new LeaderAndIsrRequest.PartitionState(0, 1, 1, brokerList, 0, brokerList, false)).asJava,
+        collection.immutable.Map(new TopicPartition(topic, 0) ->
+          new LeaderAndIsrRequest.PartitionState(0, 1, 1, brokerList, 0, brokerList, false)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       rm.becomeLeaderOrFollower(1, leaderAndIsrRequest2, (_, _) => ())
 
@@ -203,7 +206,8 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0,
-        collection.immutable.Map(new TopicPartition(topic, 0) -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
+        collection.immutable.Map(new TopicPartition(topic, 0) ->
+          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getLeaderReplicaIfLocal(new TopicPartition(topic, 0))
@@ -252,7 +256,8 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0,
-        collection.immutable.Map(new TopicPartition(topic, 0) -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
+        collection.immutable.Map(new TopicPartition(topic, 0) ->
+          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getLeaderReplicaIfLocal(new TopicPartition(topic, 0))
@@ -272,12 +277,14 @@ class ReplicaManagerTest {
       }
 
       // fetch as follower to advance the high watermark
-      fetchAsFollower(replicaManager, new TopicPartition(topic, 0), new PartitionData(numRecords, 0, 100000),
+      fetchAsFollower(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(numRecords, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_UNCOMMITTED)
 
       // fetch should return empty since LSO should be stuck at 0
       var consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0),
-        new PartitionData(0, 0, 100000), isolationLevel = IsolationLevel.READ_COMMITTED)
+        new PartitionData(0, 0, 100000, Optional.empty()),
+        isolationLevel = IsolationLevel.READ_COMMITTED)
       var fetchData = consumerFetchResult.assertFired
       assertEquals(Errors.NONE, fetchData.error)
       assertTrue(fetchData.records.batches.asScala.isEmpty)
@@ -285,7 +292,8 @@ class ReplicaManagerTest {
       assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions)
 
       // delayed fetch should timeout and return nothing
-      consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0), new PartitionData(0, 0, 100000),
+      consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(0, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_COMMITTED, minBytes = 1000)
       assertFalse(consumerFetchResult.isFired)
       timer.advanceClock(1001)
@@ -304,7 +312,8 @@ class ReplicaManagerTest {
 
       // the LSO has advanced, but the appended commit marker has not been replicated, so
       // none of the data from the transaction should be visible yet
-      consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0), new PartitionData(0, 0, 100000),
+      consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(0, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_COMMITTED)
 
       fetchData = consumerFetchResult.assertFired
@@ -312,11 +321,13 @@ class ReplicaManagerTest {
       assertTrue(fetchData.records.batches.asScala.isEmpty)
 
       // fetch as follower to advance the high watermark
-      fetchAsFollower(replicaManager, new TopicPartition(topic, 0), new PartitionData(numRecords + 1, 0, 100000),
+      fetchAsFollower(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(numRecords + 1, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_UNCOMMITTED)
 
       // now all of the records should be fetchable
-      consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0), new PartitionData(0, 0, 100000),
+      consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(0, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_COMMITTED)
 
       fetchData = consumerFetchResult.assertFired
@@ -341,7 +352,8 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0,
-        collection.immutable.Map(new TopicPartition(topic, 0) -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
+        collection.immutable.Map(new TopicPartition(topic, 0) ->
+          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, true)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
       replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       replicaManager.getLeaderReplicaIfLocal(new TopicPartition(topic, 0))
@@ -366,12 +378,14 @@ class ReplicaManagerTest {
         .onFire { response => assertEquals(Errors.NONE, response.error) }
 
       // fetch as follower to advance the high watermark
-      fetchAsFollower(replicaManager, new TopicPartition(topic, 0), new PartitionData(numRecords + 1, 0, 100000),
+      fetchAsFollower(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(numRecords + 1, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_UNCOMMITTED)
 
       // Set the minBytes in order force this request to enter purgatory. When it returns, we should still
       // see the newly aborted transaction.
-      val fetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0), new PartitionData(0, 0, 100000),
+      val fetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0),
+        new PartitionData(0, 0, 100000, Optional.empty()),
         isolationLevel = IsolationLevel.READ_COMMITTED, minBytes = 10000)
       assertFalse(fetchResult.isFired)
 
@@ -403,7 +417,8 @@ class ReplicaManagerTest {
 
       // Make this replica the leader.
       val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0,
-        collection.immutable.Map(new TopicPartition(topic, 0) -> new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, false)).asJava,
+        collection.immutable.Map(new TopicPartition(topic, 0) ->
+          new LeaderAndIsrRequest.PartitionState(0, 0, 0, brokerList, 0, brokerList, false)).asJava,
         Set(new Node(0, "host1", 0), new Node(1, "host2", 1), new Node(2, "host2", 2)).asJava).build()
       rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
       rm.getLeaderReplicaIfLocal(new TopicPartition(topic, 0))
@@ -417,13 +432,15 @@ class ReplicaManagerTest {
       }
 
       // Fetch a message above the high watermark as a follower
-      val followerFetchResult = fetchAsFollower(rm, new TopicPartition(topic, 0), new PartitionData(1, 0, 100000))
+      val followerFetchResult = fetchAsFollower(rm, new TopicPartition(topic, 0),
+        new PartitionData(1, 0, 100000, Optional.empty()))
       val followerFetchData = followerFetchResult.assertFired
       assertEquals("Should not give an exception", Errors.NONE, followerFetchData.error)
       assertTrue("Should return some data", followerFetchData.records.batches.iterator.hasNext)
 
       // Fetch a message above the high watermark as a consumer
-      val consumerFetchResult = fetchAsConsumer(rm, new TopicPartition(topic, 0), new PartitionData(1, 0, 100000))
+      val consumerFetchResult = fetchAsConsumer(rm, new TopicPartition(topic, 0),
+        new PartitionData(1, 0, 100000, Optional.empty()))
       val consumerFetchData = consumerFetchResult.assertFired
       assertEquals("Should not give an exception", Errors.NONE, consumerFetchData.error)
       assertEquals("Should return empty response", MemoryRecords.EMPTY, consumerFetchData.records)
@@ -495,8 +512,8 @@ class ReplicaManagerTest {
         fetchMaxBytes = Int.MaxValue,
         hardMaxBytesLimit = false,
         fetchInfos = Seq(
-          tp0 -> new PartitionData(1, 0, 100000),
-          tp1 -> new PartitionData(1, 0, 100000)),
+          tp0 -> new PartitionData(1, 0, 100000, Optional.empty()),
+          tp1 -> new PartitionData(1, 0, 100000, Optional.empty())),
         responseCallback = fetchCallback,
         isolationLevel = IsolationLevel.READ_UNCOMMITTED
       )
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index d91e008fe3bcc..3b7ecfb5195bc 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -15,7 +15,7 @@
 package kafka.server
 
 import java.nio.ByteBuffer
-import java.util.{Collections, LinkedHashMap, Properties}
+import java.util.{Collections, LinkedHashMap, Optional, Properties}
 import java.util.concurrent.{Executors, Future, TimeUnit}
 
 import kafka.log.LogConfig
@@ -207,7 +207,7 @@ class RequestQuotaTest extends BaseRequestTest {
 
         case ApiKeys.FETCH =>
           val partitionMap = new LinkedHashMap[TopicPartition, FetchRequest.PartitionData]
-          partitionMap.put(tp, new FetchRequest.PartitionData(0, 0, 100))
+          partitionMap.put(tp, new FetchRequest.PartitionData(0, 0, 100, Optional.of(15)))
           FetchRequest.Builder.forConsumer(0, 0, partitionMap)
 
         case ApiKeys.METADATA =>
@@ -215,11 +215,13 @@ class RequestQuotaTest extends BaseRequestTest {
 
         case ApiKeys.LIST_OFFSETS =>
           ListOffsetRequest.Builder.forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
-            .setTargetTimes(Map(tp -> (0L: java.lang.Long)).asJava)
+            .setTargetTimes(Map(tp -> new ListOffsetRequest.PartitionData(
+              0L, Optional.of[Integer](15))).asJava)
 
         case ApiKeys.LEADER_AND_ISR =>
           new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue,
-            Map(tp -> new LeaderAndIsrRequest.PartitionState(Int.MaxValue, brokerId, Int.MaxValue, List(brokerId).asJava, 2, Seq(brokerId).asJava, true)).asJava,
+            Map(tp -> new LeaderAndIsrRequest.PartitionState(Int.MaxValue, brokerId, Int.MaxValue, List(brokerId).asJava,
+              2, Seq(brokerId).asJava, true)).asJava,
             Set(new Node(brokerId, "localhost", 0)).asJava)
 
         case ApiKeys.STOP_REPLICA =>
@@ -239,7 +241,7 @@ class RequestQuotaTest extends BaseRequestTest {
 
         case ApiKeys.OFFSET_COMMIT =>
           new OffsetCommitRequest.Builder("test-group",
-            Map(tp -> new OffsetCommitRequest.PartitionData(0, "metadata")).asJava).
+            Map(tp -> new OffsetCommitRequest.PartitionData(0, 15, "metadata")).asJava).
             setMemberId("").setGenerationId(1)
 
         case ApiKeys.OFFSET_FETCH =>
@@ -290,7 +292,8 @@ class RequestQuotaTest extends BaseRequestTest {
           new InitProducerIdRequest.Builder("abc")
 
         case ApiKeys.OFFSET_FOR_LEADER_EPOCH =>
-          new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion()).add(tp, 0)
+          new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion)
+            .add(tp, Optional.of(15), 0)
 
         case ApiKeys.ADD_PARTITIONS_TO_TXN =>
           new AddPartitionsToTxnRequest.Builder("test-transactional-id", 1, 0, List(tp).asJava)
diff --git a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
index 0797b7b3d118d..d2f1c0a0ddd5c 100644
--- a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
@@ -27,7 +27,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.requests.FetchRequest.PartitionData
 import org.junit.{After, Before, Test}
-import java.util.Properties
+import java.util.{Optional, Properties}
 import java.util.concurrent.atomic.AtomicBoolean
 
 import org.apache.kafka.common.TopicPartition
@@ -63,7 +63,8 @@ class SimpleFetchTest {
   val partitionId = 0
   val topicPartition = new TopicPartition(topic, partitionId)
 
-  val fetchInfo = Seq(topicPartition -> new PartitionData(0, 0, fetchSize))
+  val fetchInfo = Seq(topicPartition -> new PartitionData(0, 0, fetchSize,
+    Optional.empty()))
 
   var replicaManager: ReplicaManager = _
 
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index 17683f4f3fd53..5ad641f11a0ed 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -16,7 +16,7 @@
   */
 package kafka.server.epoch
 
-import java.util.{Map => JMap}
+import java.util.{Optional, Map => JMap}
 
 import kafka.server.KafkaConfig._
 import kafka.server.{BlockingSend, KafkaServer, ReplicaFetcherBlockingSend}
@@ -31,7 +31,6 @@ import org.apache.kafka.common.serialization.StringSerializer
 import org.apache.kafka.common.utils.{LogContext, SystemTime}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.protocol.ApiKeys
-
 import org.junit.Assert._
 import org.junit.{After, Test}
 import org.apache.kafka.common.requests.{EpochEndOffset, OffsetsForLeaderEpochRequest, OffsetsForLeaderEpochResponse}
@@ -266,13 +265,13 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging {
   private[epoch] class TestFetcherThread(sender: BlockingSend) extends Logging {
 
     def leaderOffsetsFor(partitions: Map[TopicPartition, Int]): Map[TopicPartition, EpochEndOffset] = {
-      val request = new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(), toJavaFormat(partitions))
+      val partitionData = partitions.mapValues(
+        new OffsetsForLeaderEpochRequest.PartitionData(Optional.empty(), _))
+      val request = new OffsetsForLeaderEpochRequest.Builder(ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion,
+        partitionData.asJava)
       val response = sender.sendRequest(request)
       response.responseBody.asInstanceOf[OffsetsForLeaderEpochResponse].responses.asScala
     }
 
-    def toJavaFormat(partitions: Map[TopicPartition, Int]): JMap[TopicPartition, Integer] = {
-      partitions.map { case (tp, epoch) => tp -> epoch.asInstanceOf[Integer] }.toMap.asJava
-    }
   }
 }
diff --git a/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala b/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala
index 5c60c0017db38..4fdc4d2699232 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala
@@ -17,6 +17,7 @@
 package kafka.server.epoch
 
 import java.io.File
+import java.util.Optional
 import java.util.concurrent.atomic.AtomicBoolean
 
 import kafka.cluster.Replica
@@ -25,13 +26,12 @@ import kafka.utils.{MockTime, TestUtils}
 import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.metrics.Metrics
 import org.apache.kafka.common.protocol.Errors
-import org.apache.kafka.common.requests.EpochEndOffset
+import org.apache.kafka.common.requests.{EpochEndOffset, OffsetsForLeaderEpochRequest}
 import org.apache.kafka.common.requests.EpochEndOffset._
 import org.easymock.EasyMock._
 import org.junit.Assert._
 import org.junit.Test
 
-
 class OffsetsForLeaderEpochTest {
   private val config = TestUtils.createBrokerConfigs(1, TestUtils.MockZkConnect).map(KafkaConfig.fromProps).head
   private val time = new MockTime
@@ -43,7 +43,7 @@ class OffsetsForLeaderEpochTest {
     //Given
     val epochAndOffset = (5, 42L)
     val epochRequested: Integer = 5
-    val request = Map(tp -> epochRequested)
+    val request = Map(tp -> new OffsetsForLeaderEpochRequest.PartitionData(Optional.empty(), epochRequested))
 
     //Stubs
     val mockLog = createNiceMock(classOf[kafka.log.Log])
@@ -84,7 +84,7 @@ class OffsetsForLeaderEpochTest {
 
     //Given
     val epochRequested: Integer = 5
-    val request = Map(tp -> epochRequested)
+    val request = Map(tp -> new OffsetsForLeaderEpochRequest.PartitionData(Optional.empty(), epochRequested))
 
     //When
     val response = replicaManager.lastOffsetForLeaderEpoch(request)
@@ -106,7 +106,7 @@ class OffsetsForLeaderEpochTest {
 
     //Given
     val epochRequested: Integer = 5
-    val request = Map(tp -> epochRequested)
+    val request = Map(tp -> new OffsetsForLeaderEpochRequest.PartitionData(Optional.empty(), epochRequested))
 
     //When
     val response = replicaManager.lastOffsetForLeaderEpoch(request)

From b8651d4e82d45463d2c71798bd5852f8a605b440 Mon Sep 17 00:00:00 2001
From: Vahid Hashemian 
Date: Sun, 9 Sep 2018 14:25:30 -0700
Subject: [PATCH 0789/1847] MINOR: Code cleanup of 'clients' module (#5427)

Cleanup involves:
* Refactoring to use Java 8 constructs (lambdas,
diamond for `empty` collection methods) and library
methods (`computeIfAbsent`)
* Simplifying code (including unnecessarily complex
`equals` and `hashCode` implementations)
* Removing redundant code
* Fixing typos

Reviewers: Ryanne Dolan, Ismael Juma 
---
 .../kafka/clients/FetchSessionHandler.java    |   6 +-
 .../kafka/clients/InFlightRequests.java       |   8 +-
 .../admin/ConsumerGroupDescription.java       |   2 +-
 .../kafka/clients/admin/DeleteAclsResult.java |  45 ++--
 .../admin/internals/AdminMetadataManager.java |   7 +-
 .../kafka/clients/consumer/KafkaConsumer.java |   2 +-
 .../kafka/clients/consumer/MockConsumer.java  |  15 +-
 .../kafka/clients/consumer/RangeAssignor.java |   2 +-
 .../clients/consumer/RoundRobinAssignor.java  |   3 +-
 .../internals/AbstractPartitionAssignor.java  |   6 +-
 .../java/org/apache/kafka/common/Cluster.java |  22 +-
 .../org/apache/kafka/common/MetricName.java   |  23 +-
 .../apache/kafka/common/TopicPartition.java   |  12 +-
 .../kafka/common/TopicPartitionReplica.java   |  19 +-
 .../common/acl/AccessControlEntryFilter.java  |   4 +-
 .../kafka/common/config/AbstractConfig.java   |   4 +-
 .../apache/kafka/common/config/ConfigDef.java |  69 +++---
 .../common/config/ConfigTransformer.java      |   4 +-
 .../header/internals/RecordHeaders.java       |  28 +--
 .../common/internals/KafkaFutureImpl.java     |  10 +-
 .../common/internals/PartitionStates.java     |   6 +-
 .../memory/GarbageCollectedMemoryPool.java    |   2 +-
 .../kafka/common/network/ChannelState.java    |   3 +-
 .../kafka/common/network/KafkaChannel.java    |   6 +-
 .../network/PlaintextChannelBuilder.java      |   3 +-
 .../network/PlaintextTransportLayer.java      |  13 +-
 .../common/network/SaslChannelBuilder.java    |   2 +-
 .../kafka/common/network/Selectable.java      |  32 +--
 .../apache/kafka/common/network/Selector.java |  14 +-
 .../common/network/SslChannelBuilder.java     |   2 +-
 .../common/network/SslTransportLayer.java     |   6 +-
 .../apache/kafka/common/protocol/Errors.java  |  14 +-
 .../kafka/common/record/AbstractRecords.java  |   7 +-
 .../record/ByteBufferLogInputStream.java      |   3 +-
 .../kafka/common/record/CompressionType.java  |   2 +-
 .../kafka/common/record/FileRecords.java      |   8 +-
 .../record/KafkaLZ4BlockInputStream.java      |   6 +-
 .../kafka/common/record/MemoryRecords.java    |   7 +-
 .../common/record/MemoryRecordsBuilder.java   |   2 +-
 .../requests/AlterReplicaLogDirsRequest.java  |   2 +-
 .../common/requests/ApiVersionsRequest.java   |   4 +-
 .../requests/ControlledShutdownRequest.java   |   5 +-
 .../common/requests/CreateTopicsRequest.java  |   6 +-
 .../common/requests/DeleteAclsRequest.java    |   2 +-
 .../DescribeDelegationTokenResponse.java      |   2 +-
 .../requests/DescribeGroupsResponse.java      |  10 +-
 .../common/requests/JoinGroupRequest.java     |  26 +--
 .../common/requests/ListGroupsRequest.java    |   4 +-
 .../kafka/common/resource/ResourceFilter.java |   4 +-
 .../kafka/common/security/JaasContext.java    |   2 +-
 .../authenticator/CredentialCache.java        |   2 +-
 .../security/authenticator/LoginManager.java  |  21 +-
 .../SaslClientAuthenticator.java              |  20 +-
 .../SaslServerAuthenticator.java              |  21 +-
 .../security/kerberos/KerberosLogin.java      | 212 +++++++++---------
 .../oauthbearer/OAuthBearerLoginModule.java   |  14 +-
 .../internals/OAuthBearerSaslClient.java      |  10 +-
 .../OAuthBearerSaslClientCallbackHandler.java |   6 +-
 .../internals/OAuthBearerSaslServer.java      |   8 +-
 .../unsecured/OAuthBearerUnsecuredJws.java    |  34 ++-
 ...thBearerUnsecuredLoginCallbackHandler.java |  10 +-
 ...arerUnsecuredValidatorCallbackHandler.java |  13 +-
 .../security/plain/PlainLoginModule.java      |   9 +-
 .../plain/internals/PlainSaslServer.java      |   6 +-
 .../security/scram/ScramLoginModule.java      |   9 +-
 .../scram/internals/ScramExtensions.java      |   2 +-
 .../scram/internals/ScramSaslClient.java      |   8 +-
 .../scram/internals/ScramSaslServer.java      |   8 +-
 .../internals/ScramServerCallbackHandler.java |   3 +-
 .../kafka/common/security/ssl/SslFactory.java |   4 +-
 .../token/delegation/TokenInformation.java    |   4 +-
 .../kafka/common/utils/AppInfoParser.java     |   4 +-
 .../org/apache/kafka/common/utils/Utils.java  |   2 +-
 .../kafka/server/quota/ClientQuotaEntity.java |   4 +-
 .../common/resource/ResourceTypeTest.java     |   8 +-
 .../common/security/JaasContextTest.java      |  18 +-
 .../ClientAuthenticationFailureTest.java      |   2 +-
 .../authenticator/LoginManagerTest.java       |   2 +-
 .../authenticator/SaslAuthenticatorTest.java  |  12 +-
 .../SaslServerAuthenticatorTest.java          |   8 +-
 .../authenticator/TestDigestLoginModule.java  |   4 +-
 .../security/kerberos/KerberosNameTest.java   |   2 +-
 .../OAuthBearerLoginModuleTest.java           |  36 +--
 .../internals/OAuthBearerSaslServerTest.java  |   2 +-
 ...arerUnsecuredLoginCallbackHandlerTest.java |   5 +-
 ...UnsecuredValidatorCallbackHandlerTest.java |   8 +-
 .../OAuthBearerValidationUtilsTest.java       |   7 +-
 .../plain/internals/PlainSaslServerTest.java  |   2 +-
 .../scram/internals/ScramMessagesTest.java    |  12 +-
 .../serialization/SerializationTest.java      |   2 +-
 .../utils/ImplicitLinkedHashSetTest.java      |  10 +-
 .../kafka/common/utils/MockScheduler.java     |   2 +-
 .../kafka/common/utils/MockTimeTest.java      |   4 +-
 .../kafka/common/utils/SanitizerTest.java     |   3 +-
 .../apache/kafka/common/utils/ShellTest.java  |   2 +-
 .../apache/kafka/common/utils/UtilsTest.java  |   6 +-
 96 files changed, 472 insertions(+), 615 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
index 195324e833607..16990ac79c592 100644
--- a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
+++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java
@@ -202,7 +202,7 @@ public FetchRequestData build() {
                 next = null;
                 Map toSend =
                     Collections.unmodifiableMap(new LinkedHashMap<>(sessionPartitions));
-                return new FetchRequestData(toSend, Collections.emptyList(), toSend, nextMetadata);
+                return new FetchRequestData(toSend, Collections.emptyList(), toSend, nextMetadata);
             }
 
             List added = new ArrayList<>();
@@ -234,9 +234,7 @@ public FetchRequestData build() {
                 }
             }
             // Add any new partitions to the session.
-            for (Iterator> iter =
-                     next.entrySet().iterator(); iter.hasNext(); ) {
-                Entry entry = iter.next();
+            for (Entry entry : next.entrySet()) {
                 TopicPartition topicPartition = entry.getKey();
                 PartitionData nextData = entry.getValue();
                 if (sessionPartitions.containsKey(topicPartition)) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
index 5b7ba611714f1..efca453b6c781 100644
--- a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
+++ b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
@@ -21,7 +21,6 @@
 import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -153,12 +152,7 @@ public Iterable clearAll(String node) {
         } else {
             final Deque clearedRequests = requests.remove(node);
             inFlightRequestCount.getAndAdd(-clearedRequests.size());
-            return new Iterable() {
-                @Override
-                public Iterator iterator() {
-                    return clearedRequests.descendingIterator();
-                }
-            };
+            return () -> clearedRequests.descendingIterator();
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
index 933d66ca4b761..8947293e72e18 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
@@ -45,7 +45,7 @@ public ConsumerGroupDescription(String groupId,
                                     Node coordinator) {
         this.groupId = groupId == null ? "" : groupId;
         this.isSimpleConsumerGroup = isSimpleConsumerGroup;
-        this.members = members == null ? Collections.emptyList() :
+        this.members = members == null ? Collections.emptyList() :
             Collections.unmodifiableList(new ArrayList<>(members));
         this.partitionAssignor = partitionAssignor == null ? "" : partitionAssignor;
         this.state = state;
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteAclsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteAclsResult.java
index 19df2289f5bcc..63310bca84ea9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteAclsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteAclsResult.java
@@ -101,29 +101,26 @@ public Map> values() {
      * Note that it if the filters don't match any ACLs, this is not considered an error.
      */
     public KafkaFuture> all() {
-        return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0])).thenApply(
-            new KafkaFuture.BaseFunction>() {
-                @Override
-                public Collection apply(Void v) {
-                    List acls = new ArrayList<>();
-                    for (Map.Entry> entry : futures.entrySet()) {
-                        FilterResults results;
-                        try {
-                            results = entry.getValue().get();
-                        } catch (Throwable e) {
-                            // This should be unreachable, since the future returned by KafkaFuture#allOf should
-                            // have failed if any Future failed.
-                            throw new KafkaException("DeleteAclsResult#all: internal error", e);
-                        }
-                        for (FilterResult result : results.values()) {
-                            if (result.exception() != null) {
-                                throw result.exception();
-                            }
-                            acls.add(result.binding());
-                        }
-                    }
-                    return acls;
-                }
-            });
+        return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0])).thenApply(v -> getAclBindings(futures));
+    }
+
+    private List getAclBindings(Map> futures) {
+        List acls = new ArrayList<>();
+        for (KafkaFuture value: futures.values()) {
+            FilterResults results;
+            try {
+                results = value.get();
+            } catch (Throwable e) {
+                // This should be unreachable, since the future returned by KafkaFuture#allOf should
+                // have failed if any Future failed.
+                throw new KafkaException("DeleteAclsResult#all: internal error", e);
+            }
+            for (FilterResult result : results.values()) {
+                if (result.exception() != null)
+                    throw result.exception();
+                acls.add(result.binding());
+            }
+        }
+        return acls;
     }
 }
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminMetadataManager.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminMetadataManager.java
index 1ad3991ca24d2..3d9e5cad037e0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminMetadataManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminMetadataManager.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.clients.MetadataUpdater;
 import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.Node;
-import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.errors.AuthenticationException;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kafka.common.requests.RequestHeader;
@@ -183,9 +182,9 @@ public void clearController() {
             log.trace("Clearing cached controller node {}.", cluster.controller());
             this.cluster = new Cluster(cluster.clusterResource().clusterId(),
                 cluster.nodes(),
-                Collections.emptySet(),
-                Collections.emptySet(),
-                Collections.emptySet(),
+                Collections.emptySet(),
+                Collections.emptySet(),
+                Collections.emptySet(),
                 null);
         }
     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 33e037c3cedc7..4ea3cfd295f90 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -708,7 +708,7 @@ private KafkaConsumer(ConsumerConfig config,
             this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG),
                     true, false, clusterResourceListeners);
             List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));
-            this.metadata.update(Cluster.bootstrap(addresses), Collections.emptySet(), 0);
+            this.metadata.update(Cluster.bootstrap(addresses), Collections.emptySet(), 0);
             String metricGrpPrefix = "consumer";
             ConsumerMetrics metricsRegistry = new ConsumerMetrics(metricsTags.keySet(), "consumer");
             ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
index cf1b07fabe21b..9eee6da9d55ba 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
@@ -184,7 +184,7 @@ public synchronized ConsumerRecords poll(final Duration timeout) {
         // update the consumed offset
         final Map>> results = new HashMap<>();
         for (final TopicPartition topicPartition : records.keySet()) {
-            results.put(topicPartition, new ArrayList>());
+            results.put(topicPartition, new ArrayList<>());
         }
 
         for (Map.Entry>> entry : this.records.entrySet()) {
@@ -208,11 +208,7 @@ public synchronized void addRecord(ConsumerRecord record) {
         Set currentAssigned = new HashSet<>(this.subscriptions.assignedPartitions());
         if (!currentAssigned.contains(tp))
             throw new IllegalStateException("Cannot add records for a partition that is not assigned to the consumer");
-        List> recs = this.records.get(tp);
-        if (recs == null) {
-            recs = new ArrayList<>();
-            this.records.put(tp, recs);
-        }
+        List> recs = this.records.computeIfAbsent(tp, k -> new ArrayList<>());
         recs.add(record);
     }
 
@@ -439,12 +435,7 @@ public synchronized void schedulePollTask(Runnable task) {
     }
 
     public synchronized void scheduleNopPollTask() {
-        schedulePollTask(new Runnable() {
-            @Override
-            public void run() {
-                // noop
-            }
-        });
+        schedulePollTask(() -> { });
     }
 
     public synchronized Set paused() {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
index 5d5a2684e9a88..78956abb2182d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
@@ -64,7 +64,7 @@ public Map> assign(Map partitionsP
         Map> consumersPerTopic = consumersPerTopic(subscriptions);
         Map> assignment = new HashMap<>();
         for (String memberId : subscriptions.keySet())
-            assignment.put(memberId, new ArrayList());
+            assignment.put(memberId, new ArrayList<>());
 
         for (Map.Entry> topicEntry : consumersPerTopic.entrySet()) {
             String topic = topicEntry.getKey();
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
index 3b543f7130832..6763b828b1579 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
@@ -60,7 +60,7 @@ public Map> assign(Map partitionsP
                                                     Map subscriptions) {
         Map> assignment = new HashMap<>();
         for (String memberId : subscriptions.keySet())
-            assignment.put(memberId, new ArrayList());
+            assignment.put(memberId, new ArrayList<>());
 
         CircularIterator assigner = new CircularIterator<>(Utils.sorted(subscriptions.keySet()));
         for (TopicPartition partition : allPartitionsSorted(partitionsPerTopic, subscriptions)) {
@@ -72,7 +72,6 @@ public Map> assign(Map partitionsP
         return assignment;
     }
 
-
     public List allPartitionsSorted(Map partitionsPerTopic,
                                                     Map subscriptions) {
         SortedSet topics = new TreeSet<>();
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
index 8ec887ef3e340..35eb8eb141f93 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractPartitionAssignor.java
@@ -80,11 +80,7 @@ public void onAssignment(Assignment assignment) {
     }
 
     protected static  void put(Map> map, K key, V value) {
-        List list = map.get(key);
-        if (list == null) {
-            list = new ArrayList<>();
-            map.put(key, list);
-        }
+        List list = map.computeIfAbsent(key, k -> new ArrayList<>());
         list.add(value);
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/Cluster.java b/clients/src/main/java/org/apache/kafka/common/Cluster.java
index 33d37494bf507..24a18db0d9fb0 100644
--- a/clients/src/main/java/org/apache/kafka/common/Cluster.java
+++ b/clients/src/main/java/org/apache/kafka/common/Cluster.java
@@ -56,7 +56,7 @@ public Cluster(String clusterId,
                    Collection partitions,
                    Set unauthorizedTopics,
                    Set internalTopics) {
-        this(clusterId, false, nodes, partitions, unauthorizedTopics, Collections.emptySet(), internalTopics, null);
+        this(clusterId, false, nodes, partitions, unauthorizedTopics, Collections.emptySet(), internalTopics, null);
     }
 
     /**
@@ -70,7 +70,7 @@ public Cluster(String clusterId,
                    Set unauthorizedTopics,
                    Set internalTopics,
                    Node controller) {
-        this(clusterId, false, nodes, partitions, unauthorizedTopics, Collections.emptySet(), internalTopics, controller);
+        this(clusterId, false, nodes, partitions, unauthorizedTopics, Collections.emptySet(), internalTopics, controller);
     }
 
     /**
@@ -117,11 +117,11 @@ private Cluster(String clusterId,
         HashMap> partsForTopic = new HashMap<>();
         HashMap> partsForNode = new HashMap<>();
         for (Node n : this.nodes) {
-            partsForNode.put(n.id(), new ArrayList());
+            partsForNode.put(n.id(), new ArrayList<>());
         }
         for (PartitionInfo p : partitions) {
             if (!partsForTopic.containsKey(p.topic()))
-                partsForTopic.put(p.topic(), new ArrayList());
+                partsForTopic.put(p.topic(), new ArrayList<>());
             List psTopic = partsForTopic.get(p.topic());
             psTopic.add(p);
 
@@ -157,8 +157,8 @@ private Cluster(String clusterId,
      * Create an empty cluster instance with no nodes and no topic-partitions.
      */
     public static Cluster empty() {
-        return new Cluster(null, new ArrayList(0), new ArrayList(0), Collections.emptySet(),
-                Collections.emptySet(), null);
+        return new Cluster(null, new ArrayList<>(0), new ArrayList<>(0), Collections.emptySet(),
+            Collections.emptySet(), null);
     }
 
     /**
@@ -171,8 +171,8 @@ public static Cluster bootstrap(List addresses) {
         int nodeId = -1;
         for (InetSocketAddress address : addresses)
             nodes.add(new Node(nodeId--, address.getHostString(), address.getPort()));
-        return new Cluster(null, true, nodes, new ArrayList(0),
-                            Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), null);
+        return new Cluster(null, true, nodes, new ArrayList<>(0),
+            Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), null);
     }
 
     /**
@@ -231,7 +231,7 @@ public PartitionInfo partition(TopicPartition topicPartition) {
      */
     public List partitionsForTopic(String topic) {
         List parts = this.partitionsByTopic.get(topic);
-        return (parts == null) ? Collections.emptyList() : parts;
+        return (parts == null) ? Collections.emptyList() : parts;
     }
 
     /**
@@ -251,7 +251,7 @@ public Integer partitionCountForTopic(String topic) {
      */
     public List availablePartitionsForTopic(String topic) {
         List parts = this.availablePartitionsByTopic.get(topic);
-        return (parts == null) ? Collections.emptyList() : parts;
+        return (parts == null) ? Collections.emptyList() : parts;
     }
 
     /**
@@ -261,7 +261,7 @@ public List availablePartitionsForTopic(String topic) {
      */
     public List partitionsForNode(int nodeId) {
         List parts = this.partitionsByNode.get(nodeId);
-        return (parts == null) ? Collections.emptyList() : parts;
+        return (parts == null) ? Collections.emptyList() : parts;
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/MetricName.java b/clients/src/main/java/org/apache/kafka/common/MetricName.java
index 2136a72b0e4b8..0af77a038b4b4 100644
--- a/clients/src/main/java/org/apache/kafka/common/MetricName.java
+++ b/clients/src/main/java/org/apache/kafka/common/MetricName.java
@@ -106,9 +106,9 @@ public int hashCode() {
             return hash;
         final int prime = 31;
         int result = 1;
-        result = prime * result + ((group == null) ? 0 : group.hashCode());
-        result = prime * result + ((name == null) ? 0 : name.hashCode());
-        result = prime * result + ((tags == null) ? 0 : tags.hashCode());
+        result = prime * result + group.hashCode();
+        result = prime * result + name.hashCode();
+        result = prime * result + tags.hashCode();
         this.hash = result;
         return result;
     }
@@ -122,22 +122,7 @@ public boolean equals(Object obj) {
         if (getClass() != obj.getClass())
             return false;
         MetricName other = (MetricName) obj;
-        if (group == null) {
-            if (other.group != null)
-                return false;
-        } else if (!group.equals(other.group))
-            return false;
-        if (name == null) {
-            if (other.name != null)
-                return false;
-        } else if (!name.equals(other.name))
-            return false;
-        if (tags == null) {
-            if (other.tags != null)
-                return false;
-        } else if (!tags.equals(other.tags))
-            return false;
-        return true;
+        return group.equals(other.group) && name.equals(other.name) && tags.equals(other.tags);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/TopicPartition.java b/clients/src/main/java/org/apache/kafka/common/TopicPartition.java
index 08b2a51d58943..2c6add78aab31 100644
--- a/clients/src/main/java/org/apache/kafka/common/TopicPartition.java
+++ b/clients/src/main/java/org/apache/kafka/common/TopicPartition.java
@@ -17,6 +17,7 @@
 package org.apache.kafka.common;
 
 import java.io.Serializable;
+import java.util.Objects;
 
 /**
  * A topic name and partition number
@@ -48,7 +49,7 @@ public int hashCode() {
         final int prime = 31;
         int result = 1;
         result = prime * result + partition;
-        result = prime * result + ((topic == null) ? 0 : topic.hashCode());
+        result = prime * result + Objects.hashCode(topic);
         this.hash = result;
         return result;
     }
@@ -62,14 +63,7 @@ public boolean equals(Object obj) {
         if (getClass() != obj.getClass())
             return false;
         TopicPartition other = (TopicPartition) obj;
-        if (partition != other.partition)
-            return false;
-        if (topic == null) {
-            if (other.topic != null)
-                return false;
-        } else if (!topic.equals(other.topic))
-            return false;
-        return true;
+        return partition == other.partition && Objects.equals(topic, other.topic);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java b/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java
index 2a10439587fea..244260295a108 100644
--- a/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java
+++ b/clients/src/main/java/org/apache/kafka/common/TopicPartitionReplica.java
@@ -16,6 +16,8 @@
  */
 package org.apache.kafka.common;
 
+import org.apache.kafka.common.utils.Utils;
+
 import java.io.Serializable;
 
 
@@ -30,7 +32,7 @@ public final class TopicPartitionReplica implements Serializable {
     private final String topic;
 
     public TopicPartitionReplica(String topic, int partition, int brokerId) {
-        this.topic = topic;
+        this.topic = Utils.notNull(topic);
         this.partition = partition;
         this.brokerId = brokerId;
     }
@@ -54,7 +56,7 @@ public int hashCode() {
         }
         final int prime = 31;
         int result = 1;
-        result = prime * result + ((topic == null) ? 0 : topic.hashCode());
+        result = prime * result + topic.hashCode();
         result = prime * result + partition;
         result = prime * result + brokerId;
         this.hash = result;
@@ -70,18 +72,7 @@ public boolean equals(Object obj) {
         if (getClass() != obj.getClass())
             return false;
         TopicPartitionReplica other = (TopicPartitionReplica) obj;
-        if (partition != other.partition)
-            return false;
-        if (brokerId != other.brokerId)
-            return false;
-        if (topic == null) {
-            if (other.topic != null) {
-                return false;
-            }
-        } else if (!topic.equals(other.topic)) {
-            return false;
-        }
-        return true;
+        return partition == other.partition && brokerId == other.brokerId && topic.equals(other.topic);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/acl/AccessControlEntryFilter.java b/clients/src/main/java/org/apache/kafka/common/acl/AccessControlEntryFilter.java
index a95303e8608b6..225e73a47da78 100644
--- a/clients/src/main/java/org/apache/kafka/common/acl/AccessControlEntryFilter.java
+++ b/clients/src/main/java/org/apache/kafka/common/acl/AccessControlEntryFilter.java
@@ -109,9 +109,7 @@ public boolean matches(AccessControlEntry other) {
             return false;
         if ((operation() != AclOperation.ANY) && (!operation().equals(other.operation())))
             return false;
-        if ((permissionType() != AclPermissionType.ANY) && (!permissionType().equals(other.permissionType())))
-            return false;
-        return true;
+        return (permissionType() == AclPermissionType.ANY) || (permissionType().equals(other.permissionType()));
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
index 4f8420b3acec5..9cf13dd35df02 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java
@@ -316,7 +316,7 @@ public  T getConfiguredInstance(String key, Class t) {
      * @return The list of configured instances
      */
     public  List getConfiguredInstances(String key, Class t) {
-        return getConfiguredInstances(key, t, Collections.emptyMap());
+        return getConfiguredInstances(key, t, Collections.emptyMap());
     }
 
     /**
@@ -343,7 +343,7 @@ public  List getConfiguredInstances(String key, Class t, Map List getConfiguredInstances(List classNames, Class t, Map configOverrides) {
-        List objects = new ArrayList();
+        List objects = new ArrayList<>();
         if (classNames == null)
             return objects;
         Map configPairs = originals();
diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
index 662909ad85d9f..3868ed501e5b3 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
@@ -22,10 +22,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -187,7 +185,7 @@ public ConfigDef define(String name, Type type, Object defaultValue, Validator v
      */
     public ConfigDef define(String name, Type type, Object defaultValue, Validator validator, Importance importance, String documentation,
                             String group, int orderInGroup, Width width, String displayName, Recommender recommender) {
-        return define(name, type, defaultValue, validator, importance, documentation, group, orderInGroup, width, displayName, Collections.emptyList(), recommender);
+        return define(name, type, defaultValue, validator, importance, documentation, group, orderInGroup, width, displayName, Collections.emptyList(), recommender);
     }
 
     /**
@@ -264,7 +262,7 @@ public ConfigDef define(String name, Type type, Object defaultValue, Importance
      */
     public ConfigDef define(String name, Type type, Object defaultValue, Importance importance, String documentation,
                             String group, int orderInGroup, Width width, String displayName, Recommender recommender) {
-        return define(name, type, defaultValue, null, importance, documentation, group, orderInGroup, width, displayName, Collections.emptyList(), recommender);
+        return define(name, type, defaultValue, null, importance, documentation, group, orderInGroup, width, displayName, Collections.emptyList(), recommender);
     }
 
     /**
@@ -337,7 +335,7 @@ public ConfigDef define(String name, Type type, Importance importance, String do
      */
     public ConfigDef define(String name, Type type, Importance importance, String documentation, String group, int orderInGroup,
                             Width width, String displayName, Recommender recommender) {
-        return define(name, type, NO_DEFAULT_VALUE, null, importance, documentation, group, orderInGroup, width, displayName, Collections.emptyList(), recommender);
+        return define(name, type, NO_DEFAULT_VALUE, null, importance, documentation, group, orderInGroup, width, displayName, Collections.emptyList(), recommender);
     }
 
     /**
@@ -607,13 +605,7 @@ private void validate(String name, Map parsed, Map originalRecommendedValues = value.recommendedValues();
                 if (!originalRecommendedValues.isEmpty()) {
                     Set originalRecommendedValueSet = new HashSet<>(originalRecommendedValues);
-                    Iterator it = recommendedValues.iterator();
-                    while (it.hasNext()) {
-                        Object o = it.next();
-                        if (!originalRecommendedValueSet.contains(o)) {
-                            it.remove();
-                        }
-                    }
+                    recommendedValues.removeIf(o -> !originalRecommendedValueSet.contains(o));
                 }
                 value.recommendedValues(recommendedValues);
                 value.visible(key.recommender.visible(name, parsed));
@@ -1271,32 +1263,30 @@ private List sortedConfigs() {
         }
 
         List configs = new ArrayList<>(configKeys.values());
-        Collections.sort(configs, new Comparator() {
-            @Override
-            public int compare(ConfigKey k1, ConfigKey k2) {
-                int cmp = k1.group == null
-                        ? (k2.group == null ? 0 : -1)
-                        : (k2.group == null ? 1 : Integer.compare(groupOrd.get(k1.group), groupOrd.get(k2.group)));
-                if (cmp == 0) {
-                    cmp = Integer.compare(k1.orderInGroup, k2.orderInGroup);
-                    if (cmp == 0) {
-                        // first take anything with no default value
-                        if (!k1.hasDefault() && k2.hasDefault()) {
-                            cmp = -1;
-                        } else if (!k2.hasDefault() && k1.hasDefault()) {
-                            cmp = 1;
-                        } else {
-                            cmp = k1.importance.compareTo(k2.importance);
-                            if (cmp == 0) {
-                                return k1.name.compareTo(k2.name);
-                            }
-                        }
-                    }
+        Collections.sort(configs, (k1, k2) -> compare(k1, k2, groupOrd));
+        return configs;
+    }
+
+    private int compare(ConfigKey k1, ConfigKey k2, Map groupOrd) {
+        int cmp = k1.group == null
+            ? (k2.group == null ? 0 : -1)
+            : (k2.group == null ? 1 : Integer.compare(groupOrd.get(k1.group), groupOrd.get(k2.group)));
+        if (cmp == 0) {
+            cmp = Integer.compare(k1.orderInGroup, k2.orderInGroup);
+            if (cmp == 0) {
+                // first take anything with no default value
+                if (!k1.hasDefault() && k2.hasDefault())
+                    cmp = -1;
+                else if (!k2.hasDefault() && k1.hasDefault())
+                    cmp = 1;
+                else {
+                    cmp = k1.importance.compareTo(k2.importance);
+                    if (cmp == 0)
+                        return k1.name.compareTo(k2.name);
                 }
-                return cmp;
             }
-        });
-        return configs;
+        }
+        return cmp;
     }
 
     public void embed(final String keyPrefix, final String groupPrefix, final int startingOrd, final ConfigDef child) {
@@ -1324,12 +1314,7 @@ public void embed(final String keyPrefix, final String groupPrefix, final int st
      */
     private static Validator embeddedValidator(final String keyPrefix, final Validator base) {
         if (base == null) return null;
-        return new ConfigDef.Validator() {
-            @Override
-            public void ensureValid(String name, Object value) {
-                base.ensureValid(name.substring(keyPrefix.length()), value);
-            }
-        };
+        return (name, value) -> base.ensureValid(name.substring(keyPrefix.length()), value);
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java
index 6430ffdd41915..a830f9fe379a1 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigTransformer.java
@@ -81,7 +81,7 @@ public ConfigTransformerResult transform(Map configs) {
         // Collect the variables from the given configs that need transformation
         for (Map.Entry config : configs.entrySet()) {
             if (config.getValue() != null) {
-                List vars = getVars(config.getKey(), config.getValue(), DEFAULT_PATTERN);
+                List vars = getVars(config.getValue(), DEFAULT_PATTERN);
                 for (ConfigVariable var : vars) {
                     Map> keysByPath = keysByProvider.computeIfAbsent(var.providerName, k -> new HashMap<>());
                     Set keys = keysByPath.computeIfAbsent(var.path, k -> new HashSet<>());
@@ -121,7 +121,7 @@ public ConfigTransformerResult transform(Map configs) {
         return new ConfigTransformerResult(data, ttls);
     }
 
-    private static List getVars(String key, String value, Pattern pattern) {
+    private static List getVars(String value, Pattern pattern) {
         List configVars = new ArrayList<>();
         Matcher matcher = pattern.matcher(value);
         while (matcher.find()) {
diff --git a/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java b/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java
index 141c9720c38fc..577e758348d7a 100644
--- a/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java
+++ b/clients/src/main/java/org/apache/kafka/common/header/internals/RecordHeaders.java
@@ -28,7 +28,7 @@
 import org.apache.kafka.common.utils.AbstractIterator;
 
 public class RecordHeaders implements Headers {
-    
+
     private final List
     headers;
     private volatile boolean isReadOnly;
 
@@ -43,7 +43,7 @@ public RecordHeaders(Header[] headers) {
             this.headers = new ArrayList<>(Arrays.asList(headers));
         }
     }
-    
+
     public RecordHeaders(Iterable
     headers) {
         //Use efficient copy constructor if possible, fallback to iteration otherwise
         if (headers == null) {
@@ -99,12 +99,7 @@ public Header lastHeader(String key) {
     @Override
     public Iterable
     headers(final String key) {
         checkKey(key);
-        return new Iterable
    () {
-            @Override
-            public Iterator
     iterator() {
-                return new FilterByKeyIterator(headers.iterator(), key);
-            }
-        };
+        return () -> new FilterByKeyIterator(headers.iterator(), key);
     }
 
     @Override
@@ -119,17 +114,15 @@ public void setReadOnly() {
     public Header[] toArray() {
         return headers.isEmpty() ? Record.EMPTY_HEADERS : headers.toArray(new Header[headers.size()]);
     }
-    
+
     private void checkKey(String key) {
-        if (key == null) {
+        if (key == null)
             throw new IllegalArgumentException("key cannot be null.");
-        }
     }
-    
+
     private void canWrite() {
-        if (isReadOnly) {
+        if (isReadOnly)
             throw new IllegalStateException("RecordHeaders has been closed.");
-        }
     }
 
     private Iterator
     closeAware(final Iterator
     original) {
@@ -177,7 +170,7 @@ public String toString() {
                ", isReadOnly = " + isReadOnly +
                ')';
     }
-    
+
     private static final class FilterByKeyIterator extends AbstractIterator
     {
 
         private final Iterator
     original;
@@ -187,14 +180,13 @@ private FilterByKeyIterator(Iterator
     original, String key) {
             this.original = original;
             this.key = key;
         }
-        
+
         protected Header makeNext() {
             while (true) {
                 if (original.hasNext()) {
                     Header header = original.next();
-                    if (!header.key().equals(key)) {
+                    if (!header.key().equals(key))
                         continue;
-                    }
 
                     return header;
                 }
diff --git a/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java b/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java
index 33916ac952a5a..d61017237ee2c 100644
--- a/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java
+++ b/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java
@@ -208,7 +208,7 @@ protected synchronized void addWaiter(BiConsumer a
 
     @Override
     public synchronized boolean complete(T newValue) {
-        List> oldWaiters = null;
+        List> oldWaiters;
         synchronized (this) {
             if (done)
                 return false;
@@ -225,7 +225,7 @@ public synchronized boolean complete(T newValue) {
 
     @Override
     public boolean completeExceptionally(Throwable newException) {
-        List> oldWaiters = null;
+        List> oldWaiters;
         synchronized (this) {
             if (done)
                 return false;
@@ -257,7 +257,7 @@ public synchronized boolean cancel(boolean mayInterruptIfRunning) {
      */
     @Override
     public T get() throws InterruptedException, ExecutionException {
-        SingleWaiter waiter = new SingleWaiter();
+        SingleWaiter waiter = new SingleWaiter<>();
         addWaiter(waiter);
         return waiter.await();
     }
@@ -269,7 +269,7 @@ public T get() throws InterruptedException, ExecutionException {
     @Override
     public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException,
             TimeoutException {
-        SingleWaiter waiter = new SingleWaiter();
+        SingleWaiter waiter = new SingleWaiter<>();
         addWaiter(waiter);
         return waiter.await(timeout, unit);
     }
@@ -292,7 +292,7 @@ public synchronized T getNow(T valueIfAbsent) throws InterruptedException, Execu
      */
     @Override
     public synchronized boolean isCancelled() {
-        return (exception != null) && (exception instanceof CancellationException);
+        return exception instanceof CancellationException;
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java b/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java
index ba6563246a8c5..2918dd6c14c6d 100644
--- a/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java
+++ b/clients/src/main/java/org/apache/kafka/common/internals/PartitionStates.java
@@ -134,11 +134,7 @@ private void updateSize() {
     private void update(Map partitionToState) {
         LinkedHashMap> topicToPartitions = new LinkedHashMap<>();
         for (TopicPartition tp : partitionToState.keySet()) {
-            List partitions = topicToPartitions.get(tp.topic());
-            if (partitions == null) {
-                partitions = new ArrayList<>();
-                topicToPartitions.put(tp.topic(), partitions);
-            }
+            List partitions = topicToPartitions.computeIfAbsent(tp.topic(), k -> new ArrayList<>());
             partitions.add(tp);
         }
         for (Map.Entry> entry : topicToPartitions.entrySet()) {
diff --git a/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java b/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java
index 041d1c2f420e4..18f8ffe91c66e 100644
--- a/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java
+++ b/clients/src/main/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPool.java
@@ -77,7 +77,7 @@ protected void bufferToBeReleased(ByteBuffer justReleased) {
     }
 
     @Override
-    public void close() throws Exception {
+    public void close() {
         alive = false;
         gcListenerThread.interrupt();
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/network/ChannelState.java b/clients/src/main/java/org/apache/kafka/common/network/ChannelState.java
index 08ed1a04c94f9..2d584bd80ce92 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/ChannelState.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/ChannelState.java
@@ -62,7 +62,8 @@ public enum State {
         FAILED_SEND,
         AUTHENTICATION_FAILED,
         LOCAL_CLOSE
-    };
+    }
+
     // AUTHENTICATION_FAILED has a custom exception. For other states,
     // create a reusable `ChannelState` instance per-state.
     public static final ChannelState NOT_CONNECTED = new ChannelState(State.NOT_CONNECTED);
diff --git a/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java b/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java
index 17dc6a33ef27b..28951282a9f8c 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/KafkaChannel.java
@@ -48,7 +48,7 @@ public enum ChannelMuteState {
         MUTED_AND_RESPONSE_PENDING,
         MUTED_AND_THROTTLED,
         MUTED_AND_THROTTLED_AND_RESPONSE_PENDING
-    };
+    }
 
     /** Socket server events that will change the mute state:
      * 
      
@@ -72,7 +72,7 @@ public enum ChannelMuteEvent {
         RESPONSE_SENT,
         THROTTLE_STARTED,
         THROTTLE_ENDED
-    };
+    }
 
     private final String id;
     private final TransportLayer transportLayer;
@@ -90,7 +90,7 @@ public enum ChannelMuteEvent {
     private ChannelMuteState muteState;
     private ChannelState state;
 
-    public KafkaChannel(String id, TransportLayer transportLayer, Authenticator authenticator, int maxReceiveSize, MemoryPool memoryPool) throws IOException {
+    public KafkaChannel(String id, TransportLayer transportLayer, Authenticator authenticator, int maxReceiveSize, MemoryPool memoryPool) {
         this.id = id;
         this.transportLayer = transportLayer;
         this.authenticator = authenticator;
diff --git a/clients/src/main/java/org/apache/kafka/common/network/PlaintextChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/PlaintextChannelBuilder.java
index c7b80cbac63ff..e397f05060f06 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/PlaintextChannelBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/PlaintextChannelBuilder.java
@@ -26,7 +26,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
-import java.io.IOException;
 import java.net.InetAddress;
 import java.nio.channels.SelectionKey;
 import java.util.Map;
@@ -76,7 +75,7 @@ private PlaintextAuthenticator(Map configs, PlaintextTransportLayer t
         }
 
         @Override
-        public void authenticate() throws IOException {}
+        public void authenticate() {}
 
         @Override
         public KafkaPrincipal principal() {
diff --git a/clients/src/main/java/org/apache/kafka/common/network/PlaintextTransportLayer.java b/clients/src/main/java/org/apache/kafka/common/network/PlaintextTransportLayer.java
index ccb9c606185fa..845b1474f4e31 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/PlaintextTransportLayer.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/PlaintextTransportLayer.java
@@ -87,10 +87,9 @@ public void close() throws IOException {
     /**
      * Performs SSL handshake hence is a no-op for the non-secure
      * implementation
-     * @throws IOException
-    */
+     */
     @Override
-    public void handshake() throws IOException {}
+    public void handshake() {}
 
     /**
     * Reads a sequence of bytes from this channel into the given buffer.
@@ -121,7 +120,7 @@ public long read(ByteBuffer[] dsts) throws IOException {
      * @param dsts - The buffers into which bytes are to be transferred
      * @param offset - The offset within the buffer array of the first buffer into which bytes are to be transferred; must be non-negative and no larger than dsts.length.
      * @param length - The maximum number of buffers to be accessed; must be non-negative and no larger than dsts.length - offset
-     * @returns The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream.
+     * @return The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream.
      * @throws IOException if some other I/O error occurs
      */
     @Override
@@ -133,7 +132,7 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException {
     * Writes a sequence of bytes to this channel from the given buffer.
     *
     * @param src The buffer from which bytes are to be retrieved
-    * @returns The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream
+    * @return The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream
     * @throws IOException If some other I/O error occurs
     */
     @Override
@@ -145,7 +144,7 @@ public int write(ByteBuffer src) throws IOException {
     * Writes a sequence of bytes to this channel from the given buffer.
     *
     * @param srcs The buffer from which bytes are to be retrieved
-    * @returns The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream
+    * @return The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream
     * @throws IOException If some other I/O error occurs
     */
     @Override
@@ -180,7 +179,7 @@ public boolean hasPendingWrites() {
      * Returns ANONYMOUS as Principal.
      */
     @Override
-    public Principal peerPrincipal() throws IOException {
+    public Principal peerPrincipal() {
         return principal;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java
index ca34b71a39618..172e629b7d544 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java
@@ -156,7 +156,7 @@ public void configure(Map configs) throws KafkaException {
 
     @Override
     public Set reconfigurableConfigs() {
-        return securityProtocol == SecurityProtocol.SASL_SSL ? SslConfigs.RECONFIGURABLE_CONFIGS : Collections.emptySet();
+        return securityProtocol == SecurityProtocol.SASL_SSL ? SslConfigs.RECONFIGURABLE_CONFIGS : Collections.emptySet();
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java
index efb603c8dc361..8f81dbeebe969 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java
@@ -30,7 +30,7 @@ public interface Selectable {
     /**
      * See {@link #connect(String, InetSocketAddress, int, int) connect()}
      */
-    public static final int USE_DEFAULT_BUFFER_SIZE = -1;
+    int USE_DEFAULT_BUFFER_SIZE = -1;
 
     /**
      * Begin establishing a socket connection to the given address identified by the given address
@@ -40,83 +40,83 @@ public interface Selectable {
      * @param receiveBufferSize The receive buffer for the socket
      * @throws IOException If we cannot begin connecting
      */
-    public void connect(String id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException;
+    void connect(String id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException;
 
     /**
      * Wakeup this selector if it is blocked on I/O
      */
-    public void wakeup();
+    void wakeup();
 
     /**
      * Close this selector
      */
-    public void close();
+    void close();
 
     /**
      * Close the connection identified by the given id
      */
-    public void close(String id);
+    void close(String id);
 
     /**
      * Queue the given request for sending in the subsequent {@link #poll(long) poll()} calls
      * @param send The request to send
      */
-    public void send(Send send);
+    void send(Send send);
 
     /**
      * Do I/O. Reads, writes, connection establishment, etc.
      * @param timeout The amount of time to block if there is nothing to do
      * @throws IOException
      */
-    public void poll(long timeout) throws IOException;
+    void poll(long timeout) throws IOException;
 
     /**
      * The list of sends that completed on the last {@link #poll(long) poll()} call.
      */
-    public List completedSends();
+    List completedSends();
 
     /**
      * The list of receives that completed on the last {@link #poll(long) poll()} call.
      */
-    public List completedReceives();
+    List completedReceives();
 
     /**
      * The connections that finished disconnecting on the last {@link #poll(long) poll()}
      * call. Channel state indicates the local channel state at the time of disconnection.
      */
-    public Map disconnected();
+    Map disconnected();
 
     /**
      * The list of connections that completed their connection on the last {@link #poll(long) poll()}
      * call.
      */
-    public List connected();
+    List connected();
 
     /**
      * Disable reads from the given connection
      * @param id The id for the connection
      */
-    public void mute(String id);
+    void mute(String id);
 
     /**
      * Re-enable reads from the given connection
      * @param id The id for the connection
      */
-    public void unmute(String id);
+    void unmute(String id);
 
     /**
      * Disable reads from all connections
      */
-    public void muteAll();
+    void muteAll();
 
     /**
      * Re-enable reads from all connections
      */
-    public void unmuteAll();
+    void unmuteAll();
 
     /**
      * returns true  if a channel is ready
      * @param id The id for the connection
      */
-    public boolean isChannelReady(String id);
+    boolean isChannelReady(String id);
 }
diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
index 806bda700e30d..44223e7ab31e2 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java
@@ -20,8 +20,6 @@
 import org.apache.kafka.common.MetricName;
 import org.apache.kafka.common.errors.AuthenticationException;
 import org.apache.kafka.common.memory.MemoryPool;
-import org.apache.kafka.common.metrics.Measurable;
-import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -227,7 +225,7 @@ public Selector(int maxReceiveSize,
     }
 
     public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, ChannelBuilder channelBuilder, LogContext logContext) {
-        this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, Collections.emptyMap(), true, channelBuilder, logContext);
+        this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, Collections.emptyMap(), true, channelBuilder, logContext);
     }
 
     public Selector(long connectionMaxIdleMS, int failedAuthenticationDelayMs, Metrics metrics, Time time, String metricGrpPrefix, ChannelBuilder channelBuilder, LogContext logContext) {
@@ -552,7 +550,7 @@ void pollSelectionKeys(Set selectionKeys,
 
                 /* if channel is ready write to any sockets that have space in their buffer and for which we have data */
                 if (channel.ready() && key.isWritable()) {
-                    Send send = null;
+                    Send send;
                     try {
                         send = channel.write();
                     } catch (Exception e) {
@@ -919,7 +917,7 @@ private boolean hasStagedReceives() {
      */
     private void addToStagedReceives(KafkaChannel channel, NetworkReceive receive) {
         if (!stagedReceives.containsKey(channel))
-            stagedReceives.put(channel, new ArrayDeque());
+            stagedReceives.put(channel, new ArrayDeque<>());
 
         Deque deque = stagedReceives.get(channel);
         deque.add(receive);
@@ -1045,11 +1043,7 @@ public SelectorMetrics(Metrics metrics, String metricGrpPrefix, Map channels.size());
         }
 
         private Meter createMeter(Metrics metrics, String groupName, Map metricTags,
diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
index b43e4c3a472fa..86d41d08518df 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java
@@ -163,7 +163,7 @@ private SslAuthenticator(Map configs, SslTransportLayer transportLaye
          * No-Op for plaintext authenticator
          */
         @Override
-        public void authenticate() throws IOException {}
+        public void authenticate() {}
 
         /**
          * Constructs Principal using configured principalBuilder.
diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
index 08a39e71d50ba..917e5a211f707 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/SslTransportLayer.java
@@ -76,7 +76,7 @@ public static SslTransportLayer create(String channelId, SelectionKey key, SSLEn
     }
 
     // Prefer `create`, only use this in tests
-    SslTransportLayer(String channelId, SelectionKey key, SSLEngine sslEngine) throws IOException {
+    SslTransportLayer(String channelId, SelectionKey key, SSLEngine sslEngine) {
         this.channelId = channelId;
         this.key = key;
         this.socketChannel = (SocketChannel) key.channel();
@@ -372,7 +372,7 @@ private void doHandshake() throws IOException {
         }
     }
 
-    private SSLHandshakeException renegotiationException() throws IOException {
+    private SSLHandshakeException renegotiationException() {
         return new SSLHandshakeException("Renegotiation is not supported");
     }
 
@@ -715,7 +715,7 @@ public long write(ByteBuffer[] srcs) throws IOException {
      * SSLSession's peerPrincipal for the remote host.
      * @return Principal
      */
-    public Principal peerPrincipal() throws IOException {
+    public Principal peerPrincipal() {
         try {
             return sslEngine.getSession().getPeerPrincipal();
         } catch (SSLPeerUnverifiedException se) {
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index d4610602d1e66..a796ec912cb2a 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -111,7 +111,7 @@
  * Do not add exceptions that occur only on the client or only on the server here.
  */
 public enum Errors {
-    UNKNOWN_SERVER_ERROR(-1, "The server experienced an unexpected error when processing the request,",
+    UNKNOWN_SERVER_ERROR(-1, "The server experienced an unexpected error when processing the request.",
             UnknownServerException::new),
     NONE(0, null, message -> null),
     OFFSET_OUT_OF_RANGE(1, "The requested offset is not within the range of offsets maintained by the server.",
@@ -159,8 +159,8 @@ public enum Errors {
     ILLEGAL_GENERATION(22, "Specified group generation id is not valid.",
             IllegalGenerationException::new),
     INCONSISTENT_GROUP_PROTOCOL(23,
-            "The group member's supported protocols are incompatible with those of existing members" +
-                " or first group member tried to join with empty protocol type or empty protocol list.",
+            "The group member's supported protocols are incompatible with those of existing members " +
+            "or first group member tried to join with empty protocol type or empty protocol list.",
             InconsistentGroupProtocolException::new),
     INVALID_GROUP_ID(24, "The configured groupId is invalid.",
             InvalidGroupIdException::new),
@@ -201,8 +201,8 @@ public enum Errors {
     NOT_CONTROLLER(41, "This is not the correct controller for this cluster.",
             NotControllerException::new),
     INVALID_REQUEST(42, "This most likely occurs because of a request being malformed by the " +
-                "client library or the message was sent to an incompatible broker. See the broker logs " +
-                "for more details.",
+            "client library or the message was sent to an incompatible broker. See the broker logs " +
+            "for more details.",
             InvalidRequestException::new),
     UNSUPPORTED_FOR_MESSAGE_FORMAT(43, "The message format version on the broker does not support the request.",
             UnsupportedForMessageFormatException::new),
@@ -221,10 +221,10 @@ public enum Errors {
             "its transactional id.",
             InvalidPidMappingException::new),
     INVALID_TRANSACTION_TIMEOUT(50, "The transaction timeout is larger than the maximum value allowed by " +
-                "the broker (as configured by transaction.max.timeout.ms).",
+            "the broker (as configured by transaction.max.timeout.ms).",
             InvalidTxnTimeoutException::new),
     CONCURRENT_TRANSACTIONS(51, "The producer attempted to update a transaction " +
-                "while another concurrent operation on the same transaction was ongoing.",
+            "while another concurrent operation on the same transaction was ongoing.",
             ConcurrentTransactionsException::new),
     TRANSACTION_COORDINATOR_FENCED(52, "Indicates that the transaction coordinator sending a WriteTxnMarker " +
             "is no longer the current coordinator for a given producer.",
diff --git a/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java b/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java
index 5e41901fb8843..1994a71968a1c 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/AbstractRecords.java
@@ -25,12 +25,7 @@
 
 public abstract class AbstractRecords implements Records {
 
-    private final Iterable records = new Iterable() {
-        @Override
-        public Iterator iterator() {
-            return recordsIterator();
-        }
-    };
+    private final Iterable records = this::recordsIterator;
 
     @Override
     public boolean hasMatchingMagic(byte magic) {
diff --git a/clients/src/main/java/org/apache/kafka/common/record/ByteBufferLogInputStream.java b/clients/src/main/java/org/apache/kafka/common/record/ByteBufferLogInputStream.java
index 7f91f266158f6..572abd8e812c7 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/ByteBufferLogInputStream.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/ByteBufferLogInputStream.java
@@ -18,7 +18,6 @@
 
 import org.apache.kafka.common.errors.CorruptRecordException;
 
-import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import static org.apache.kafka.common.record.Records.HEADER_SIZE_UP_TO_MAGIC;
@@ -39,7 +38,7 @@ class ByteBufferLogInputStream implements LogInputStream {
         this.maxMessageSize = maxMessageSize;
     }
 
-    public MutableRecordBatch nextBatch() throws IOException {
+    public MutableRecordBatch nextBatch() {
         int remaining = buffer.remaining();
 
         Integer batchSize = nextBatchSize();
diff --git a/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java b/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
index 9b3bfc45983a9..a333d2aea2799 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/CompressionType.java
@@ -128,7 +128,7 @@ public InputStream wrapForInput(ByteBuffer inputBuffer, byte messageVersion, Buf
     /**
      * Wrap bufferStream with an OutputStream that will compress data with this CompressionType.
      *
-     * Note: Unlike {@link #wrapForInput}, {@link #wrapForOutput} cannot take {@#link ByteBuffer}s directly.
+     * Note: Unlike {@link #wrapForInput}, {@link #wrapForOutput} cannot take {@link ByteBuffer}s directly.
      * Currently, {@link MemoryRecordsBuilder#writeDefaultBatchHeader()} and {@link MemoryRecordsBuilder#writeLegacyCompressedWrapperHeader()}
      * write to the underlying buffer in the given {@link ByteBufferOutputStream} after the compressed data has been written.
      * In the event that the buffer needs to be expanded while writing the data, access to the underlying buffer needs to be preserved.
diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
index cebb5fa09d1a7..3537fc34bb018 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
@@ -32,7 +32,6 @@
 import java.nio.channels.GatheringByteChannel;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
-import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -374,12 +373,7 @@ public String toString() {
      * @return An iterator over batches starting from {@code start}
      */
     public Iterable batchesFrom(final int start) {
-        return new Iterable() {
-            @Override
-            public Iterator iterator() {
-                return batchIterator(start);
-            }
-        };
+        return () -> batchIterator(start);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockInputStream.java b/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockInputStream.java
index 56f10581db409..850b1e96e55f2 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockInputStream.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/KafkaLZ4BlockInputStream.java
@@ -264,12 +264,12 @@ public long skip(long n) throws IOException {
     }
 
     @Override
-    public int available() throws IOException {
+    public int available() {
         return decompressedBuffer == null ? 0 : decompressedBuffer.remaining();
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() {
         bufferSupplier.release(decompressionBuffer);
     }
 
@@ -279,7 +279,7 @@ public void mark(int readlimit) {
     }
 
     @Override
-    public void reset() throws IOException {
+    public void reset() {
         throw new RuntimeException("reset not supported");
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
index af62e09c57ecd..c6db18d419162 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
@@ -47,12 +47,7 @@ public class MemoryRecords extends AbstractRecords {
 
     private final ByteBuffer buffer;
 
-    private final Iterable batches = new Iterable() {
-        @Override
-        public Iterator iterator() {
-            return batchIterator();
-        }
-    };
+    private final Iterable batches = this::batchIterator;
 
     private int validBytes = -1;
 
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
index 6f6404fa2d959..1c7a6c746dfe2 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
@@ -41,7 +41,7 @@ public class MemoryRecordsBuilder {
     private static final float COMPRESSION_RATE_ESTIMATION_FACTOR = 1.05f;
     private static final DataOutputStream CLOSED_STREAM = new DataOutputStream(new OutputStream() {
         @Override
-        public void write(int b) throws IOException {
+        public void write(int b) {
             throw new IllegalStateException("MemoryRecordsBuilder is closed for record appends");
         }
     });
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java
index 03bc0987eff00..b7cf1d8d40bee 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java
@@ -117,7 +117,7 @@ protected Struct toStruct() {
         Map> dirPartitions = new HashMap<>();
         for (Map.Entry entry: partitionDirs.entrySet()) {
             if (!dirPartitions.containsKey(entry.getValue()))
-                dirPartitions.put(entry.getValue(), new ArrayList());
+                dirPartitions.put(entry.getValue(), new ArrayList<>());
             dirPartitions.get(entry.getValue()).add(entry.getKey());
         }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
index e154ac92bb70c..03f385b9ba586 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
@@ -95,10 +95,10 @@ public ApiVersionsResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         short version = version();
         switch (version) {
             case 0:
-                return new ApiVersionsResponse(Errors.forException(e), Collections.emptyList());
+                return new ApiVersionsResponse(Errors.forException(e), Collections.emptyList());
             case 1:
             case 2:
-                return new ApiVersionsResponse(throttleTimeMs, Errors.forException(e), Collections.emptyList());
+                return new ApiVersionsResponse(throttleTimeMs, Errors.forException(e), Collections.emptyList());
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
                         version, this.getClass().getSimpleName(), ApiKeys.API_VERSIONS.latestVersion()));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
index d6db1e1dc0a94..dc86dd7053003 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.requests;
 
-import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.protocol.types.Field;
@@ -79,10 +78,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         switch (versionId) {
             case 0:
             case 1:
-                return new ControlledShutdownResponse(Errors.forException(e), Collections.emptySet());
+                return new ControlledShutdownResponse(Errors.forException(e), Collections.emptySet());
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
-                        versionId, this.getClass().getSimpleName(), ApiKeys.CONTROLLED_SHUTDOWN.latestVersion()));
+                    versionId, this.getClass().getSimpleName(), ApiKeys.CONTROLLED_SHUTDOWN.latestVersion()));
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
index 9f05c5a5c0a58..16d4c97b5522c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
@@ -120,12 +120,12 @@ private TopicDetails(int numPartitions,
         public TopicDetails(int partitions,
                             short replicationFactor,
                             Map configs) {
-            this(partitions, replicationFactor, Collections.>emptyMap(), configs);
+            this(partitions, replicationFactor, Collections.emptyMap(), configs);
         }
 
         public TopicDetails(int partitions,
                             short replicationFactor) {
-            this(partitions, replicationFactor, Collections.emptyMap());
+            this(partitions, replicationFactor, Collections.emptyMap());
         }
 
         public TopicDetails(Map> replicasAssignments,
@@ -134,7 +134,7 @@ public TopicDetails(Map> replicasAssignments,
         }
 
         public TopicDetails(Map> replicasAssignments) {
-            this(replicasAssignments, Collections.emptyMap());
+            this(replicasAssignments, Collections.emptyMap());
         }
 
         @Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java
index c3fc194cd378a..2ed7c24f68a8f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java
@@ -140,7 +140,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable
                 List responses = new ArrayList<>();
                 for (int i = 0; i < filters.size(); i++) {
                     responses.add(new DeleteAclsResponse.AclFilterResponse(
-                        ApiError.fromThrowable(throwable), Collections.emptySet()));
+                        ApiError.fromThrowable(throwable), Collections.emptySet()));
                 }
                 return new DeleteAclsResponse(throttleTimeMs, responses);
             default:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java
index c86b14131932c..64ed27bfd0003 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java
@@ -82,7 +82,7 @@ public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error, List());
+        this(throttleTimeMs, error, new ArrayList<>());
     }
 
     public DescribeDelegationTokenResponse(Struct struct) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
index fbdfaa3200a5c..06c247156a1ea 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
@@ -196,11 +196,11 @@ public List members() {
 
         public static GroupMetadata forError(Errors error) {
             return new DescribeGroupsResponse.GroupMetadata(
-                    error,
-                    DescribeGroupsResponse.UNKNOWN_STATE,
-                    DescribeGroupsResponse.UNKNOWN_PROTOCOL_TYPE,
-                    DescribeGroupsResponse.UNKNOWN_PROTOCOL,
-                    Collections.emptyList());
+                error,
+                DescribeGroupsResponse.UNKNOWN_STATE,
+                DescribeGroupsResponse.UNKNOWN_PROTOCOL_TYPE,
+                DescribeGroupsResponse.UNKNOWN_PROTOCOL,
+                Collections.emptyList());
         }
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
index 366509dbab1f7..76cc1a3cd54c3 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
@@ -201,22 +201,22 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
             case 0:
             case 1:
                 return new JoinGroupResponse(
-                        Errors.forException(e),
-                        JoinGroupResponse.UNKNOWN_GENERATION_ID,
-                        JoinGroupResponse.UNKNOWN_PROTOCOL,
-                        JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
-                        JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
-                        Collections.emptyMap());
+                    Errors.forException(e),
+                    JoinGroupResponse.UNKNOWN_GENERATION_ID,
+                    JoinGroupResponse.UNKNOWN_PROTOCOL,
+                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
+                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
+                    Collections.emptyMap());
             case 2:
             case 3:
                 return new JoinGroupResponse(
-                        throttleTimeMs,
-                        Errors.forException(e),
-                        JoinGroupResponse.UNKNOWN_GENERATION_ID,
-                        JoinGroupResponse.UNKNOWN_PROTOCOL,
-                        JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
-                        JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
-                        Collections.emptyMap());
+                    throttleTimeMs,
+                    Errors.forException(e),
+                    JoinGroupResponse.UNKNOWN_GENERATION_ID,
+                    JoinGroupResponse.UNKNOWN_PROTOCOL,
+                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
+                    JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
+                    Collections.emptyMap());
 
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
index 27254cb1f2404..baab1e1c56ca4 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
@@ -71,10 +71,10 @@ public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         short versionId = version();
         switch (versionId) {
             case 0:
-                return new ListGroupsResponse(Errors.forException(e), Collections.emptyList());
+                return new ListGroupsResponse(Errors.forException(e), Collections.emptyList());
             case 1:
             case 2:
-                return new ListGroupsResponse(throttleTimeMs, Errors.forException(e), Collections.emptyList());
+                return new ListGroupsResponse(throttleTimeMs, Errors.forException(e), Collections.emptyList());
             default:
                 throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
                         versionId, this.getClass().getSimpleName(), ApiKeys.LIST_GROUPS.latestVersion()));
diff --git a/clients/src/main/java/org/apache/kafka/common/resource/ResourceFilter.java b/clients/src/main/java/org/apache/kafka/common/resource/ResourceFilter.java
index 0a4611f9874b2..2ea9032e61941 100644
--- a/clients/src/main/java/org/apache/kafka/common/resource/ResourceFilter.java
+++ b/clients/src/main/java/org/apache/kafka/common/resource/ResourceFilter.java
@@ -93,9 +93,7 @@ public int hashCode() {
     public boolean matches(Resource other) {
         if ((name != null) && (!name.equals(other.name())))
             return false;
-        if ((resourceType != ResourceType.ANY) && (!resourceType.equals(other.resourceType())))
-            return false;
-        return true;
+        return (resourceType == ResourceType.ANY) || (resourceType.equals(other.resourceType()));
     }
 
     /**
diff --git a/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java b/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java
index 849a97894db28..231ee167110b3 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java
@@ -140,7 +140,7 @@ private static JaasContext defaultContext(JaasContext.Type contextType, String l
      * The type of the SASL login context, it should be SERVER for the broker and CLIENT for the clients (consumer, producer,
      * etc.). This is used to validate behaviour (e.g. some functionality is only available in the broker or clients).
      */
-    public enum Type { CLIENT, SERVER; }
+    public enum Type { CLIENT, SERVER }
 
     private final String name;
     private final Type type;
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/CredentialCache.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/CredentialCache.java
index 96e7426afe836..28923abbfce99 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/CredentialCache.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/CredentialCache.java
@@ -23,7 +23,7 @@ public class CredentialCache {
     private final ConcurrentHashMap> cacheMap = new ConcurrentHashMap<>();
 
     public  Cache createCache(String mechanism, Class credentialClass) {
-        Cache cache = new Cache(credentialClass);
+        Cache cache = new Cache<>(credentialClass);
         Cache oldCache = (Cache) cacheMap.putIfAbsent(mechanism, cache);
         return oldCache == null ? cache : oldCache;
     }
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java
index 2bc44c5c5b731..7bb7be7c72d5c 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java
@@ -16,16 +16,6 @@
  */
 package org.apache.kafka.common.security.authenticator;
 
-import javax.security.auth.Subject;
-import javax.security.auth.login.LoginException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-
 import org.apache.kafka.common.config.ConfigException;
 import org.apache.kafka.common.config.SaslConfigs;
 import org.apache.kafka.common.config.types.Password;
@@ -39,6 +29,13 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.security.auth.Subject;
+import javax.security.auth.login.LoginException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
 public class LoginManager {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(LoginManager.class);
@@ -55,7 +52,7 @@ public class LoginManager {
     private int refCount;
 
     private LoginManager(JaasContext jaasContext, String saslMechanism, Map configs,
-                         LoginMetadata loginMetadata) throws IOException, LoginException {
+                         LoginMetadata loginMetadata) throws LoginException {
         this.loginMetadata = loginMetadata;
         this.login = Utils.newInstance(loginMetadata.loginClass);
         loginCallbackHandler = Utils.newInstance(loginMetadata.loginCallbackClass);
@@ -89,7 +86,7 @@ private LoginManager(JaasContext jaasContext, String saslMechanism, Map defaultLoginClass,
-                                                   Map configs) throws IOException, LoginException {
+                                                   Map configs) throws LoginException {
         Class loginClass = configuredClassOrDefault(configs, jaasContext,
                 saslMechanism, SaslConfigs.SASL_LOGIN_CLASS, defaultLoginClass);
         Class defaultLoginCallbackHandlerClass = OAuthBearerLoginModule.OAUTHBEARER_MECHANISM
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 8934e8e548702..fb74f5faa95e4 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -113,7 +113,7 @@ public SaslClientAuthenticator(Map configs,
                                    String host,
                                    String mechanism,
                                    boolean handshakeRequestEnable,
-                                   TransportLayer transportLayer) throws IOException {
+                                   TransportLayer transportLayer) {
         this.node = node;
         this.subject = subject;
         this.callbackHandler = callbackHandler;
@@ -144,13 +144,11 @@ public SaslClientAuthenticator(Map configs,
 
     private SaslClient createSaslClient() {
         try {
-            return Subject.doAs(subject, new PrivilegedExceptionAction() {
-                public SaslClient run() throws SaslException {
-                    String[] mechs = {mechanism};
-                    LOG.debug("Creating SaslClient: client={};service={};serviceHostname={};mechs={}",
-                        clientPrincipalName, servicePrincipal, host, Arrays.toString(mechs));
-                    return Sasl.createSaslClient(mechs, clientPrincipalName, servicePrincipal, host, configs, callbackHandler);
-                }
+            return Subject.doAs(subject, (PrivilegedExceptionAction) () -> {
+                String[] mechs = {mechanism};
+                LOG.debug("Creating SaslClient: client={};service={};serviceHostname={};mechs={}",
+                    clientPrincipalName, servicePrincipal, host, Arrays.toString(mechs));
+                return Sasl.createSaslClient(mechs, clientPrincipalName, servicePrincipal, host, configs, callbackHandler);
             });
         } catch (PrivilegedActionException e) {
             throw new SaslAuthenticationException("Failed to create SaslClient with mechanism " + mechanism, e.getCause());
@@ -354,11 +352,7 @@ private byte[] createSaslToken(final byte[] saslToken, boolean isInitial) throws
             if (isInitial && !saslClient.hasInitialResponse())
                 return saslToken;
             else
-                return Subject.doAs(subject, new PrivilegedExceptionAction() {
-                    public byte[] run() throws SaslException {
-                        return saslClient.evaluateChallenge(saslToken);
-                    }
-                });
+                return Subject.doAs(subject, (PrivilegedExceptionAction) () -> saslClient.evaluateChallenge(saslToken));
         } catch (PrivilegedActionException e) {
             String error = "An error: (" + e + ") occurred when evaluating SASL token received from the Kafka Broker.";
             KerberosError kerberosError = KerberosError.fromException(e);
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
index 43cb0a448707b..48a49fe4e946c 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
@@ -129,7 +129,7 @@ public SaslServerAuthenticator(Map configs,
                                    KerberosShortNamer kerberosNameParser,
                                    ListenerName listenerName,
                                    SecurityProtocol securityProtocol,
-                                   TransportLayer transportLayer) throws IOException {
+                                   TransportLayer transportLayer) {
         this.callbackHandlers = callbackHandlers;
         this.connectionId = connectionId;
         this.subjects = subjects;
@@ -164,12 +164,8 @@ private void createSaslServer(String mechanism) throws IOException {
             saslServer = createSaslKerberosServer(callbackHandler, configs, subject);
         } else {
             try {
-                saslServer = Subject.doAs(subject, new PrivilegedExceptionAction() {
-                    public SaslServer run() throws SaslException {
-                        return Sasl.createSaslServer(saslMechanism, "kafka", serverAddress().getHostName(),
-                                configs, callbackHandler);
-                    }
-                });
+                saslServer = Subject.doAs(subject, (PrivilegedExceptionAction) () ->
+                    Sasl.createSaslServer(saslMechanism, "kafka", serverAddress().getHostName(), configs, callbackHandler));
             } catch (PrivilegedActionException e) {
                 throw new SaslException("Kafka Server failed to create a SaslServer to interact with a client during session authentication", e.getCause());
             }
@@ -212,11 +208,8 @@ private SaslServer createSaslKerberosServer(final AuthenticateCallbackHandler sa
         }
 
         try {
-            return Subject.doAs(subject, new PrivilegedExceptionAction() {
-                public SaslServer run() throws SaslException {
-                    return Sasl.createSaslServer(saslMechanism, servicePrincipalName, serviceHostname, configs, saslServerCallbackHandler);
-                }
-            });
+            return Subject.doAs(subject, (PrivilegedExceptionAction) () ->
+                    Sasl.createSaslServer(saslMechanism, servicePrincipalName, serviceHostname, configs, saslServerCallbackHandler));
         } catch (PrivilegedActionException e) {
             throw new SaslException("Kafka Server failed to create a SaslServer to interact with a client during session authentication", e.getCause());
         }
@@ -308,11 +301,11 @@ public void close() throws IOException {
             saslServer.dispose();
     }
 
-    private void setSaslState(SaslState saslState) throws IOException {
+    private void setSaslState(SaslState saslState) {
         setSaslState(saslState, null);
     }
 
-    private void setSaslState(SaslState saslState, AuthenticationException exception) throws IOException {
+    private void setSaslState(SaslState saslState, AuthenticationException exception) {
         if (netOutBuffer != null && !netOutBuffer.completed()) {
             pendingSaslState = saslState;
             pendingException = exception;
diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java
index ec996a8ccc6c0..2faf6bef8babc 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java
@@ -134,127 +134,125 @@ public LoginContext login() throws LoginException {
         // TGT's existing expiry date and the configured minTimeBeforeRelogin. For testing and development,
         // you can decrease the interval of expiration of tickets (for example, to 3 minutes) by running:
         //  "modprinc -maxlife 3mins " in kadmin.
-        t = KafkaThread.daemon(String.format("kafka-kerberos-refresh-thread-%s", principal), new Runnable() {
-            public void run() {
-                log.info("[Principal={}]: TGT refresh thread started.", principal);
-                while (true) {  // renewal thread's main loop. if it exits from here, thread will exit.
-                    KerberosTicket tgt = getTGT();
-                    long now = currentWallTime();
-                    long nextRefresh;
-                    Date nextRefreshDate;
-                    if (tgt == null) {
-                        nextRefresh = now + minTimeBeforeRelogin;
-                        nextRefreshDate = new Date(nextRefresh);
-                        log.warn("[Principal={}]: No TGT found: will try again at {}", principal, nextRefreshDate);
+        t = KafkaThread.daemon(String.format("kafka-kerberos-refresh-thread-%s", principal), () -> {
+            log.info("[Principal={}]: TGT refresh thread started.", principal);
+            while (true) {  // renewal thread's main loop. if it exits from here, thread will exit.
+                KerberosTicket tgt = getTGT();
+                long now = currentWallTime();
+                long nextRefresh;
+                Date nextRefreshDate;
+                if (tgt == null) {
+                    nextRefresh = now + minTimeBeforeRelogin;
+                    nextRefreshDate = new Date(nextRefresh);
+                    log.warn("[Principal={}]: No TGT found: will try again at {}", principal, nextRefreshDate);
+                } else {
+                    nextRefresh = getRefreshTime(tgt);
+                    long expiry = tgt.getEndTime().getTime();
+                    Date expiryDate = new Date(expiry);
+                    if (isUsingTicketCache && tgt.getRenewTill() != null && tgt.getRenewTill().getTime() < expiry) {
+                        log.warn("The TGT cannot be renewed beyond the next expiry date: {}." +
+                            "This process will not be able to authenticate new SASL connections after that " +
+                            "time (for example, it will not be able to authenticate a new connection with a Kafka " +
+                            "Broker).  Ask your system administrator to either increase the " +
+                            "'renew until' time by doing : 'modprinc -maxrenewlife {} ' within " +
+                            "kadmin, or instead, to generate a keytab for {}. Because the TGT's " +
+                            "expiry cannot be further extended by refreshing, exiting refresh thread now.",
+                            expiryDate, principal, principal);
+                        return;
+                    }
+                    // determine how long to sleep from looking at ticket's expiry.
+                    // We should not allow the ticket to expire, but we should take into consideration
+                    // minTimeBeforeRelogin. Will not sleep less than minTimeBeforeRelogin, unless doing so
+                    // would cause ticket expiration.
+                    if ((nextRefresh > expiry) || (now + minTimeBeforeRelogin > expiry)) {
+                        // expiry is before next scheduled refresh).
+                        log.info("[Principal={}]: Refreshing now because expiry is before next scheduled refresh time.", principal);
+                        nextRefresh = now;
                     } else {
-                        nextRefresh = getRefreshTime(tgt);
-                        long expiry = tgt.getEndTime().getTime();
-                        Date expiryDate = new Date(expiry);
-                        if (isUsingTicketCache && tgt.getRenewTill() != null && tgt.getRenewTill().getTime() < expiry) {
-                            log.warn("The TGT cannot be renewed beyond the next expiry date: {}." +
-                                    "This process will not be able to authenticate new SASL connections after that " +
-                                    "time (for example, it will not be able to authenticate a new connection with a Kafka " +
-                                    "Broker).  Ask your system administrator to either increase the " +
-                                    "'renew until' time by doing : 'modprinc -maxrenewlife {} ' within " +
-                                    "kadmin, or instead, to generate a keytab for {}. Because the TGT's " +
-                                    "expiry cannot be further extended by refreshing, exiting refresh thread now.",
-                                    expiryDate, principal, principal);
-                            return;
-                        }
-                        // determine how long to sleep from looking at ticket's expiry.
-                        // We should not allow the ticket to expire, but we should take into consideration
-                        // minTimeBeforeRelogin. Will not sleep less than minTimeBeforeRelogin, unless doing so
-                        // would cause ticket expiration.
-                        if ((nextRefresh > expiry) || (now + minTimeBeforeRelogin > expiry)) {
-                            // expiry is before next scheduled refresh).
-                            log.info("[Principal={}]: Refreshing now because expiry is before next scheduled refresh time.", principal);
-                            nextRefresh = now;
-                        } else {
-                            if (nextRefresh < (now + minTimeBeforeRelogin)) {
-                                // next scheduled refresh is sooner than (now + MIN_TIME_BEFORE_LOGIN).
-                                Date until = new Date(nextRefresh);
-                                Date newUntil = new Date(now + minTimeBeforeRelogin);
-                                log.warn("[Principal={}]: TGT refresh thread time adjusted from {} to {} since the former is sooner " +
-                                        "than the minimum refresh interval ({} seconds) from now.",
-                                        principal, until, newUntil, minTimeBeforeRelogin / 1000);
-                            }
-                            nextRefresh = Math.max(nextRefresh, now + minTimeBeforeRelogin);
-                        }
-                        nextRefreshDate = new Date(nextRefresh);
-                        if (nextRefresh > expiry) {
-                            log.error("[Principal={}]: Next refresh: {} is later than expiry {}. This may indicate a clock skew problem." +
-                                    "Check that this host and the KDC hosts' clocks are in sync. Exiting refresh thread.",
-                                    principal, nextRefreshDate, expiryDate);
-                            return;
+                        if (nextRefresh < (now + minTimeBeforeRelogin)) {
+                            // next scheduled refresh is sooner than (now + MIN_TIME_BEFORE_LOGIN).
+                            Date until = new Date(nextRefresh);
+                            Date newUntil = new Date(now + minTimeBeforeRelogin);
+                            log.warn("[Principal={}]: TGT refresh thread time adjusted from {} to {} since the former is sooner " +
+                                "than the minimum refresh interval ({} seconds) from now.",
+                                principal, until, newUntil, minTimeBeforeRelogin / 1000);
                         }
+                        nextRefresh = Math.max(nextRefresh, now + minTimeBeforeRelogin);
                     }
-                    if (now < nextRefresh) {
-                        Date until = new Date(nextRefresh);
-                        log.info("[Principal={}]: TGT refresh sleeping until: {}", principal, until);
-                        try {
-                            Thread.sleep(nextRefresh - now);
-                        } catch (InterruptedException ie) {
-                            log.warn("[Principal={}]: TGT renewal thread has been interrupted and will exit.", principal);
-                            return;
-                        }
-                    } else {
-                        log.error("[Principal={}]: NextRefresh: {} is in the past: exiting refresh thread. Check"
-                                + " clock sync between this host and KDC - (KDC's clock is likely ahead of this host)."
-                                + " Manual intervention will be required for this client to successfully authenticate."
-                                + " Exiting refresh thread.", principal, nextRefreshDate);
+                    nextRefreshDate = new Date(nextRefresh);
+                    if (nextRefresh > expiry) {
+                        log.error("[Principal={}]: Next refresh: {} is later than expiry {}. This may indicate a clock skew problem." +
+                            "Check that this host and the KDC hosts' clocks are in sync. Exiting refresh thread.",
+                            principal, nextRefreshDate, expiryDate);
+                        return;
+                    }
+                }
+                if (now < nextRefresh) {
+                    Date until = new Date(nextRefresh);
+                    log.info("[Principal={}]: TGT refresh sleeping until: {}", principal, until);
+                    try {
+                        Thread.sleep(nextRefresh - now);
+                    } catch (InterruptedException ie) {
+                        log.warn("[Principal={}]: TGT renewal thread has been interrupted and will exit.", principal);
                         return;
                     }
-                    if (isUsingTicketCache) {
-                        String kinitArgs = "-R";
-                        int retry = 1;
-                        while (retry >= 0) {
-                            try {
-                                log.debug("[Principal={}]: Running ticket cache refresh command: {} {}", principal, kinitCmd, kinitArgs);
-                                Shell.execCommand(kinitCmd, kinitArgs);
-                                break;
-                            } catch (Exception e) {
-                                if (retry > 0) {
-                                    --retry;
-                                    // sleep for 10 seconds
-                                    try {
-                                        Thread.sleep(10 * 1000);
-                                    } catch (InterruptedException ie) {
-                                        log.error("[Principal={}]: Interrupted while renewing TGT, exiting Login thread", principal);
-                                        return;
-                                    }
-                                } else {
-                                    log.warn("[Principal={}]: Could not renew TGT due to problem running shell command: '{} {}'. " +
-                                            "Exiting refresh thread.", principal, kinitCmd, kinitArgs, e);
+                } else {
+                    log.error("[Principal={}]: NextRefresh: {} is in the past: exiting refresh thread. Check"
+                        + " clock sync between this host and KDC - (KDC's clock is likely ahead of this host)."
+                        + " Manual intervention will be required for this client to successfully authenticate."
+                        + " Exiting refresh thread.", principal, nextRefreshDate);
+                    return;
+                }
+                if (isUsingTicketCache) {
+                    String kinitArgs = "-R";
+                    int retry = 1;
+                    while (retry >= 0) {
+                        try {
+                            log.debug("[Principal={}]: Running ticket cache refresh command: {} {}", principal, kinitCmd, kinitArgs);
+                            Shell.execCommand(kinitCmd, kinitArgs);
+                            break;
+                        } catch (Exception e) {
+                            if (retry > 0) {
+                                --retry;
+                                // sleep for 10 seconds
+                                try {
+                                    Thread.sleep(10 * 1000);
+                                } catch (InterruptedException ie) {
+                                    log.error("[Principal={}]: Interrupted while renewing TGT, exiting Login thread", principal);
                                     return;
                                 }
+                            } else {
+                                log.warn("[Principal={}]: Could not renew TGT due to problem running shell command: '{} {}'. " +
+                                    "Exiting refresh thread.", principal, kinitCmd, kinitArgs, e);
+                                return;
                             }
                         }
                     }
-                    try {
-                        int retry = 1;
-                        while (retry >= 0) {
-                            try {
-                                reLogin();
-                                break;
-                            } catch (LoginException le) {
-                                if (retry > 0) {
-                                    --retry;
-                                    // sleep for 10 seconds.
-                                    try {
-                                        Thread.sleep(10 * 1000);
-                                    } catch (InterruptedException e) {
-                                        log.error("[Principal={}]: Interrupted during login retry after LoginException:", principal, le);
-                                        throw le;
-                                    }
-                                } else {
-                                    log.error("[Principal={}]: Could not refresh TGT.", principal, le);
+                }
+                try {
+                    int retry = 1;
+                    while (retry >= 0) {
+                        try {
+                            reLogin();
+                            break;
+                        } catch (LoginException le) {
+                            if (retry > 0) {
+                                --retry;
+                                // sleep for 10 seconds.
+                                try {
+                                    Thread.sleep(10 * 1000);
+                                } catch (InterruptedException e) {
+                                    log.error("[Principal={}]: Interrupted during login retry after LoginException:", principal, le);
+                                    throw le;
                                 }
+                            } else {
+                                log.error("[Principal={}]: Could not refresh TGT.", principal, le);
                             }
                         }
-                    } catch (LoginException le) {
-                        log.error("[Principal={}]: Failed to refresh TGT: refresh thread exiting now.", principal, le);
-                        return;
                     }
+                } catch (LoginException le) {
+                    log.error("[Principal={}]: Failed to refresh TGT: refresh thread exiting now.", principal, le);
+                    return;
                 }
             }
         });
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerLoginModule.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerLoginModule.java
index ac273ccbb0c03..1dcd1991aedde 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerLoginModule.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/OAuthBearerLoginModule.java
@@ -121,14 +121,14 @@
  * 
      
  * Here is a typical, basic JAAS configuration for a client leveraging unsecured
  * SASL/OAUTHBEARER authentication:
- * 
+ *
  * 
        * KafkaClient {
  *      org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule Required
  *      unsecuredLoginStringClaim_sub="thePrincipalName";
  * };
  * 
      
- * 
+ *
  * An implementation of the {@link Login} interface specific to the
  * {@code OAUTHBEARER} mechanism is automatically applied; it periodically
  * refreshes any token before it expires so that the client can continue to make
@@ -196,14 +196,14 @@
  * 
      
  * Here is a typical, basic JAAS configuration for a broker leveraging unsecured
  * SASL/OAUTHBEARER validation:
- * 
+ *
  * 
        * KafkaServer {
  *      org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule Required
  *      unsecuredLoginStringClaim_sub="thePrincipalName";
  * };
  * 
      
- * 
+ *
  * Production use cases will require writing an implementation of
  * {@link AuthenticateCallbackHandler} that can handle an instance of
  * {@link OAuthBearerValidatorCallback} and declaring it via the
@@ -229,7 +229,7 @@
  * Better to mitigate this possibility by leaving the existing token (which
  * still has some lifetime left) in place until a new replacement token is
  * actually retrieved. This implementation supports this.
- * 
+ *
  * @see SaslConfigs#SASL_LOGIN_REFRESH_WINDOW_FACTOR_DOC
  * @see SaslConfigs#SASL_LOGIN_REFRESH_WINDOW_JITTER_DOC
  * @see SaslConfigs#SASL_LOGIN_REFRESH_MIN_PERIOD_SECONDS_DOC
@@ -350,7 +350,7 @@ public boolean logout() {
     }
 
     @Override
-    public boolean commit() throws LoginException {
+    public boolean commit() {
         if (tokenRequiringCommit == null) {
             if (log.isDebugEnabled())
                 log.debug("Nothing here to commit");
@@ -371,7 +371,7 @@ public boolean commit() throws LoginException {
     }
 
     @Override
-    public boolean abort() throws LoginException {
+    public boolean abort() {
         if (tokenRequiringCommit != null) {
             log.info("Login aborted");
             tokenRequiringCommit = null;
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClient.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClient.java
index 16db3c8b38223..e32e16d5a0928 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClient.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClient.java
@@ -58,7 +58,7 @@ public class OAuthBearerSaslClient implements SaslClient {
 
     enum State {
         SEND_CLIENT_FIRST_MESSAGE, RECEIVE_SERVER_FIRST_MESSAGE, RECEIVE_SERVER_MESSAGE_AFTER_FAILURE, COMPLETE, FAILED
-    };
+    }
 
     private State state;
 
@@ -127,14 +127,14 @@ public boolean isComplete() {
     }
 
     @Override
-    public byte[] unwrap(byte[] incoming, int offset, int len) throws SaslException {
+    public byte[] unwrap(byte[] incoming, int offset, int len) {
         if (!isComplete())
             throw new IllegalStateException("Authentication exchange has not completed");
         return Arrays.copyOfRange(incoming, offset, offset + len);
     }
 
     @Override
-    public byte[] wrap(byte[] outgoing, int offset, int len) throws SaslException {
+    public byte[] wrap(byte[] outgoing, int offset, int len) {
         if (!isComplete())
             throw new IllegalStateException("Authentication exchange has not completed");
         return Arrays.copyOfRange(outgoing, offset, offset + len);
@@ -148,7 +148,7 @@ public Object getNegotiatedProperty(String propName) {
     }
 
     @Override
-    public void dispose() throws SaslException {
+    public void dispose() {
     }
 
     private void setState(State state) {
@@ -173,7 +173,7 @@ private SaslExtensions retrieveCustomExtensions() throws SaslException {
     public static class OAuthBearerSaslClientFactory implements SaslClientFactory {
         @Override
         public SaslClient createSaslClient(String[] mechanisms, String authorizationId, String protocol,
-                String serverName, Map props, CallbackHandler callbackHandler) throws SaslException {
+                String serverName, Map props, CallbackHandler callbackHandler) {
             String[] mechanismNamesCompatibleWithPolicy = getMechanismNames(props);
             for (String mechanism : mechanisms) {
                 for (int i = 0; i < mechanismNamesCompatibleWithPolicy.length; i++) {
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientCallbackHandler.java
index ab2b71632562d..352e6b6d9dec7 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientCallbackHandler.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslClientCallbackHandler.java
@@ -53,7 +53,7 @@ public class OAuthBearerSaslClientCallbackHandler implements AuthenticateCallbac
 
     /**
      * Return true if this instance has been configured, otherwise false
-     * 
+     *
      * @return true if this instance has been configured, otherwise false
      */
     public boolean configured() {
@@ -91,8 +91,8 @@ private void handleCallback(OAuthBearerTokenCallback callback) throws IOExceptio
             throw new IllegalArgumentException("Callback had a token already");
         Subject subject = Subject.getSubject(AccessController.getContext());
         Set privateCredentials = subject != null
-                ? subject.getPrivateCredentials(OAuthBearerToken.class)
-                : Collections.emptySet();
+            ? subject.getPrivateCredentials(OAuthBearerToken.class)
+            : Collections.emptySet();
         if (privateCredentials.size() != 1)
             throw new IOException(
                     String.format("Unable to find OAuth Bearer token in Subject's private credentials (size=%d)",
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslServer.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslServer.java
index 8f89c147328c7..db332b4153517 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslServer.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/OAuthBearerSaslServer.java
@@ -128,21 +128,21 @@ public boolean isComplete() {
     }
 
     @Override
-    public byte[] unwrap(byte[] incoming, int offset, int len) throws SaslException {
+    public byte[] unwrap(byte[] incoming, int offset, int len) {
         if (!complete)
             throw new IllegalStateException("Authentication exchange has not completed");
         return Arrays.copyOfRange(incoming, offset, offset + len);
     }
 
     @Override
-    public byte[] wrap(byte[] outgoing, int offset, int len) throws SaslException {
+    public byte[] wrap(byte[] outgoing, int offset, int len) {
         if (!complete)
             throw new IllegalStateException("Authentication exchange has not completed");
         return Arrays.copyOfRange(outgoing, offset, offset + len);
     }
 
     @Override
-    public void dispose() throws SaslException {
+    public void dispose() {
         complete = false;
         tokenForNegotiatedProperty = null;
         extensions = null;
@@ -225,7 +225,7 @@ public static String[] mechanismNamesCompatibleWithPolicy(Map props)
     public static class OAuthBearerSaslServerFactory implements SaslServerFactory {
         @Override
         public SaslServer createSaslServer(String mechanism, String protocol, String serverName, Map props,
-                CallbackHandler callbackHandler) throws SaslException {
+                CallbackHandler callbackHandler) {
             String[] mechanismNamesCompatibleWithPolicy = getMechanismNames(props);
             for (int i = 0; i < mechanismNamesCompatibleWithPolicy.length; i++) {
                 if (mechanismNamesCompatibleWithPolicy[i].equals(mechanism)) {
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java
index b764e540e93c6..b5b301650ef4f 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java
@@ -35,13 +35,12 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeType;
-import com.fasterxml.jackson.databind.node.NumericNode;
 
 /**
  * A simple unsecured JWS implementation. The '{@code nbf}' claim is ignored if
  * it is given because the related logic is not required for Kafka testing and
  * development purposes.
- * 
+ *
  * @see RFC 7515
  */
 public class OAuthBearerUnsecuredJws implements OAuthBearerToken {
@@ -58,7 +57,7 @@ public class OAuthBearerUnsecuredJws implements OAuthBearerToken {
 
     /**
      * Constructor with the given principal and scope claim names
-     * 
+     *
      * @param compactSerialization
      *            the compact serialization to parse as an unsecured JWS
      * @param principalClaimName
@@ -118,7 +117,7 @@ public String value() {
 
     /**
      * Return the 3 or 5 dot-separated sections of the JWT compact serialization
-     * 
+     *
      * @return the 3 or 5 dot-separated sections of the JWT compact serialization
      */
     public List splits() {
@@ -127,7 +126,7 @@ public List splits() {
 
     /**
      * Return the JOSE Header as a {@code Map}
-     * 
+     *
      * @return the JOSE header
      */
     public Map header() {
@@ -156,7 +155,7 @@ public Set scope() throws OAuthBearerIllegalTokenException {
 
     /**
      * Return the JWT Claim Set as a {@code Map}
-     * 
+     *
      * @return the (always non-null but possibly empty) claims
      */
     public Map claims() {
@@ -165,7 +164,7 @@ public Map claims() {
 
     /**
      * Return the (always non-null/non-empty) principal claim name
-     * 
+     *
      * @return the (always non-null/non-empty) principal claim name
      */
     public String principalClaimName() {
@@ -174,7 +173,7 @@ public String principalClaimName() {
 
     /**
      * Return the (always non-null/non-empty) scope claim name
-     * 
+     *
      * @return the (always non-null/non-empty) scope claim name
      */
     public String scopeClaimName() {
@@ -183,7 +182,7 @@ public String scopeClaimName() {
 
     /**
      * Indicate if the claim exists and is the given type
-     * 
+     *
      * @param claimName
      *            the mandatory JWT claim name
      * @param type
@@ -205,7 +204,7 @@ public boolean isClaimType(String claimName, Class type) {
 
     /**
      * Extract a claim of the given type
-     * 
+     *
      * @param claimName
      *            the mandatory JWT claim name
      * @param type
@@ -228,7 +227,7 @@ public  T claim(String claimName, Class type) throws OAuthBearerIllegalTok
 
     /**
      * Extract a claim in its raw form
-     * 
+     *
      * @param claimName
      *            the mandatory JWT claim name
      * @return the raw claim value, if it exists, otherwise null
@@ -241,7 +240,7 @@ public Object rawClaim(String claimName) {
      * Return the
      * Expiration
      * Time claim
-     * 
+     *
      * @return the Expiration
      *         Time claim if available, otherwise null
@@ -255,7 +254,7 @@ public Number expirationTime() throws OAuthBearerIllegalTokenException {
     /**
      * Return the Issued
      * At claim
-     * 
+     *
      * @return the
      *         Issued
      *         At claim if available, otherwise null
@@ -269,7 +268,7 @@ public Number issuedAt() throws OAuthBearerIllegalTokenException {
     /**
      * Return the
      * Subject claim
-     * 
+     *
      * @return the Subject claim
      *         if available, otherwise null
@@ -284,7 +283,7 @@ public String subject() throws OAuthBearerIllegalTokenException {
      * Decode the given Base64URL-encoded value, parse the resulting JSON as a JSON
      * object, and return the map of member names to their values (each value being
      * represented as either a String, a Number, or a List of Strings).
-     * 
+     *
      * @param split
      *            the value to decode and parse
      * @return the map of JSON member names to their String, Number, or String List
@@ -326,12 +325,11 @@ private List extractCompactSerializationSplits() {
     private static Object convert(JsonNode value) {
         if (value.isArray()) {
             List retvalList = new ArrayList<>();
-            for (JsonNode arrayElement : value) {
+            for (JsonNode arrayElement : value)
                 retvalList.add(arrayElement.asText());
-            }
             return retvalList;
         }
-        return value.getNodeType() == JsonNodeType.NUMBER ? ((NumericNode) value).numberValue() : value.asText();
+        return value.getNodeType() == JsonNodeType.NUMBER ? value.numberValue() : value.asText();
     }
 
     private Long calculateStartTimeMs() throws OAuthBearerIllegalTokenException {
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java
index 83b2602f5c42f..8d259e30895b1 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java
@@ -79,7 +79,7 @@
  * be something other than '{@code scope}'
  * 
    
  * For example:
- * 
+ *
  * 
      * KafkaClient {
  *      org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule Required
@@ -89,7 +89,7 @@
  *      unsecuredLoginExtension_traceId="123";
  * };
  * 
    
- * 
+ *
  * This class is the default when the SASL mechanism is OAUTHBEARER and no value
  * is explicitly set via either the {@code sasl.login.callback.handler.class}
  * client configuration property or the
@@ -122,7 +122,7 @@ public class OAuthBearerUnsecuredLoginCallbackHandler implements AuthenticateCal
 
     /**
      * For testing
-     * 
+     *
      * @param time
      *            the mandatory time to set
      */
@@ -132,7 +132,7 @@ void time(Time time) {
 
     /**
      * Return true if this instance has been configured, otherwise false
-     * 
+     *
      * @return true if this instance has been configured, otherwise false
      */
     public boolean configured() {
@@ -179,7 +179,7 @@ public void close() {
         // empty
     }
 
-    private void handleTokenCallback(OAuthBearerTokenCallback callback) throws IOException {
+    private void handleTokenCallback(OAuthBearerTokenCallback callback) {
         if (callback.token() != null)
             throw new IllegalArgumentException("Callback had a token already");
         String principalClaimNameValue = optionValue(PRINCIPAL_CLAIM_NAME_OPTION);
diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java
index 2e21cf43f4edc..54e289c2c5621 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java
@@ -16,7 +16,6 @@
  */
 package org.apache.kafka.common.security.oauthbearer.internals.unsecured;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -59,7 +58,7 @@
  * clock skew (the default is 0)
  *